From 4dec9bb5c86125a2734cfe59d3d5911fc3d5b33d Mon Sep 17 00:00:00 2001 From: Celine Pelletier Date: Fri, 14 Jun 2024 13:24:57 -0400 Subject: [PATCH] :boom: clean legacy stuff --- .env schema | 3 - src/env.ts | 1 - src/reports/biospecimen-data/configKf.ts | 66 +++--- src/reports/biospecimen-data/configKfNext.ts | 55 ----- src/reports/biospecimen-data/index.ts | 7 +- src/reports/clinical-data/configKf.ts | 200 ++++++---------- src/reports/clinical-data/configKfNext.ts | 123 ---------- src/reports/clinical-data/index.ts | 7 +- src/reports/family-clinical-data/configKf.ts | 121 +++++----- .../family-clinical-data/configKfNext.ts | 118 ---------- src/reports/family-clinical-data/index.ts | 7 +- src/utils/riffClient.ts | 42 ---- src/utils/riffError.ts | 12 - src/utils/setsTypes.ts | 12 - src/utils/sqonUtils.test.ts | 217 ------------------ src/utils/sqonUtils.ts | 13 +- 16 files changed, 167 insertions(+), 837 deletions(-) delete mode 100644 src/reports/biospecimen-data/configKfNext.ts delete mode 100644 src/reports/clinical-data/configKfNext.ts delete mode 100644 src/reports/family-clinical-data/configKfNext.ts delete mode 100644 src/utils/riffClient.ts delete mode 100644 src/utils/riffError.ts diff --git a/.env schema b/.env schema index 766f916..b3888f8 100644 --- a/.env schema +++ b/.env schema @@ -14,9 +14,6 @@ KEYCLOAK_URL= KEYCLOAK_REALM= KEYCLOAK_CLIENT= -# Riff -RIFF_URL= - # Users-API USERS_API_URL= diff --git a/src/env.ts b/src/env.ts index 21b77c8..a2fc230 100644 --- a/src/env.ts +++ b/src/env.ts @@ -27,5 +27,4 @@ export const KEYCLOAK_URL = process.env.KEYCLOAK_URL || 'https://kf-keycloak-qa. export const KEYCLOAK_REALM = process.env.KEYCLOAK_REALM || 'kidsfirstdrc'; export const KEYCLOAK_CLIENT = process.env.KEYCLOAK_CLIENT || 'kidsfirst-apis'; -export const RIFF_URL = process.env.RIFF_URL || 'https://riff-qa.kf-strides.org'; export const USERS_API_URL = process.env.USERS_API_URL || 'https://users-api-qa.373997854230.d3b.io'; diff --git a/src/reports/biospecimen-data/configKf.ts b/src/reports/biospecimen-data/configKf.ts index 4004db3..01f5e74 100644 --- a/src/reports/biospecimen-data/configKf.ts +++ b/src/reports/biospecimen-data/configKf.ts @@ -1,42 +1,42 @@ +import { esBiospecimenIndex } from '../../env'; import { QueryConfig, ReportConfig, SheetConfig } from '../types'; const biospecimens: SheetConfig = { sheetName: 'Biospecimens', - root: 'biospecimens', + root: null, columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'biospecimens.kf_id' }, - { field: 'biospecimens.external_sample_id' }, - { field: 'biospecimens.external_aliquot_id' }, - { field: 'biospecimens.age_at_event_days' }, - { field: 'biospecimens.composition' }, - { field: 'biospecimens.analyte_type' }, - { field: 'biospecimens.concentration_mg_per_ml' }, - { field: 'biospecimens.volume_ul' }, - { field: 'biospecimens.shipment_date' }, - { field: 'biospecimens.shipment_origin' }, - { field: 'biospecimens.sequencing_center_id' }, - { field: 'biospecimens.consent_type' }, - { field: 'biospecimens.dbgap_consent_code' }, - { field: 'biospecimens.method_of_sample_procurement' }, - { field: 'biospecimens.source_text_tumor_descriptor' }, - { field: 'biospecimens.ncit_id_anatomical_site' }, - { field: 'biospecimens.ncit_id_tissue_type' }, - { field: 'biospecimens.source_text_anatomical_site' }, - { field: 'biospecimens.source_text_tissue_type' }, - { field: 'biospecimens.uberon_id_anatomical_site' }, - { field: 'biospecimens.spatial_descriptor' }, + { field: 'participant.participant_id', header: 'Participant ID' }, + { field: 'participant.external_id', header: 'External Participant ID' }, + { field: 'collection_sample_id', header: 'Collection ID' }, + { field: 'external_collection_sample_id', header: 'External Collection ID' }, + { field: 'collection_sample_type', header: 'Collection Sample Type' }, + { field: 'sample_id', header: 'Sample ID' }, + { field: 'external_sample_id', header: 'External Sample ID' }, + { field: 'sample_type', header: 'Sample Type' }, + { field: 'parent_sample_id', header: 'Parent Sample ID' }, + { field: 'parent_sample_type', header: 'Parent Sample Type' }, + { field: 'study.study_code', header: 'Study Code' }, + { field: 'age_at_biospecimen_collection', header: 'Age at Biospecimen Collection (Days)' }, + { field: 'status', header: 'Sample Availability' }, + { field: 'volume', header: 'Volume' }, + { field: 'volume_unit', header: 'Volume Unit' }, + { field: 'collection_method_of_sample_procurement', header: 'Method of Sample Procurement' }, + { field: 'diagnoses.mondo_display_term', header: 'Histological Diagnosis (MONDO)' }, + { field: 'diagnoses.diagnosis_ncit', header: 'Histological Diagnosis (NCIT)' }, + { field: 'diagnoses.source_text', header: 'Histological Diagnosis (Source Text)' }, + { field: 'diagnoses.source_text_tumor_location', header: 'Tumor Location (Source Text)' }, + //FIXME { field: '', header: 'Tumor Descriptor (Source Text)' }, + { field: 'collection_ncit_anatomy_site_id', header: 'Anatomical Site (NCIT)' }, + { field: 'collection_anatomy_site', header: 'Anatomical Site (Source Text)' }, + // TODO: Add this back when it's ready { field: 'ncit_id_tissue_type', header: 'Tissue Type (NCIT)' }, + // TODO: Add this back when it's ready { field: 'tissue_type_source_text', header: 'Tissue Type (Source Text)' }, + { field: 'consent_type', header: 'Consent Type' }, + { field: 'dbgap_consent_code', header: 'dbGaP Consent Code' }, + { field: 'files.sequencing_experiment.sequencing_center_id', header: 'Sequencing Center ID' }, ], sort: [ - // TODO : SORT BY biospecimens.kf_id { - 'biospecimens.kf_id': { - order: 'asc', - }, - }, - { - kf_id: { + 'participant.participant_id': { order: 'asc', }, }, @@ -44,8 +44,8 @@ const biospecimens: SheetConfig = { }; const queryConfigs: QueryConfig = { - indexName: 'participant', - alias: 'participant_centric', + indexName: 'biospecimen', + alias: esBiospecimenIndex, }; const sheetConfigs: SheetConfig[] = [biospecimens]; diff --git a/src/reports/biospecimen-data/configKfNext.ts b/src/reports/biospecimen-data/configKfNext.ts deleted file mode 100644 index 01f5e74..0000000 --- a/src/reports/biospecimen-data/configKfNext.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { esBiospecimenIndex } from '../../env'; -import { QueryConfig, ReportConfig, SheetConfig } from '../types'; - -const biospecimens: SheetConfig = { - sheetName: 'Biospecimens', - root: null, - columns: [ - { field: 'participant.participant_id', header: 'Participant ID' }, - { field: 'participant.external_id', header: 'External Participant ID' }, - { field: 'collection_sample_id', header: 'Collection ID' }, - { field: 'external_collection_sample_id', header: 'External Collection ID' }, - { field: 'collection_sample_type', header: 'Collection Sample Type' }, - { field: 'sample_id', header: 'Sample ID' }, - { field: 'external_sample_id', header: 'External Sample ID' }, - { field: 'sample_type', header: 'Sample Type' }, - { field: 'parent_sample_id', header: 'Parent Sample ID' }, - { field: 'parent_sample_type', header: 'Parent Sample Type' }, - { field: 'study.study_code', header: 'Study Code' }, - { field: 'age_at_biospecimen_collection', header: 'Age at Biospecimen Collection (Days)' }, - { field: 'status', header: 'Sample Availability' }, - { field: 'volume', header: 'Volume' }, - { field: 'volume_unit', header: 'Volume Unit' }, - { field: 'collection_method_of_sample_procurement', header: 'Method of Sample Procurement' }, - { field: 'diagnoses.mondo_display_term', header: 'Histological Diagnosis (MONDO)' }, - { field: 'diagnoses.diagnosis_ncit', header: 'Histological Diagnosis (NCIT)' }, - { field: 'diagnoses.source_text', header: 'Histological Diagnosis (Source Text)' }, - { field: 'diagnoses.source_text_tumor_location', header: 'Tumor Location (Source Text)' }, - //FIXME { field: '', header: 'Tumor Descriptor (Source Text)' }, - { field: 'collection_ncit_anatomy_site_id', header: 'Anatomical Site (NCIT)' }, - { field: 'collection_anatomy_site', header: 'Anatomical Site (Source Text)' }, - // TODO: Add this back when it's ready { field: 'ncit_id_tissue_type', header: 'Tissue Type (NCIT)' }, - // TODO: Add this back when it's ready { field: 'tissue_type_source_text', header: 'Tissue Type (Source Text)' }, - { field: 'consent_type', header: 'Consent Type' }, - { field: 'dbgap_consent_code', header: 'dbGaP Consent Code' }, - { field: 'files.sequencing_experiment.sequencing_center_id', header: 'Sequencing Center ID' }, - ], - sort: [ - { - 'participant.participant_id': { - order: 'asc', - }, - }, - ], -}; - -const queryConfigs: QueryConfig = { - indexName: 'biospecimen', - alias: esBiospecimenIndex, -}; - -const sheetConfigs: SheetConfig[] = [biospecimens]; - -const reportConfig: ReportConfig = { queryConfigs, sheetConfigs }; - -export default reportConfig; diff --git a/src/reports/biospecimen-data/index.ts b/src/reports/biospecimen-data/index.ts index 606be83..6c78a2d 100644 --- a/src/reports/biospecimen-data/index.ts +++ b/src/reports/biospecimen-data/index.ts @@ -10,20 +10,17 @@ import generateReport from '../generateReport'; import { ProjectType } from '../types'; import configInclude from './configInclude'; import configKf from './configKf'; -import configKfNext from './configKfNext'; const biospecimenDataReport = async (req: Request, res: Response): Promise => { console.time('biospecimen-data'); - const { sqon, projectId, filename = null, isKfNext = false } = req.body; + const { sqon, projectId, filename = null } = req.body; const userId = req['kauth']?.grant?.access_token?.content?.sub; const accessToken = req.headers.authorization; const p = PROJECT.toLowerCase().trim(); let reportConfig; - if (isKfNext) { - reportConfig = configKfNext; - } else if (p === ProjectType.include) { + if (p === ProjectType.include) { reportConfig = configInclude; } else if (p === ProjectType.kidsFirst) { reportConfig = configKf; diff --git a/src/reports/clinical-data/configKf.ts b/src/reports/clinical-data/configKf.ts index 526be6a..00c02b6 100644 --- a/src/reports/clinical-data/configKf.ts +++ b/src/reports/clinical-data/configKf.ts @@ -1,46 +1,30 @@ import { QueryConfig, ReportConfig, SheetConfig } from '../types'; -// TODO : comment all the fields to document it, also put it in README.md const participants: SheetConfig = { sheetName: 'Participants', root: null, columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'family_id' }, - { field: 'is_proband' }, - { field: 'study.short_name' }, - { field: 'family.family_compositions.composition' }, - { field: 'diagnosis_category' }, - { field: 'gender' }, - { field: 'race' }, - { field: 'ethnicity' }, - { field: 'outcome.vital_status' }, - { - field: 'outcome.age_at_event_days', - header: 'Age at the Last Vital Status (Days)', - }, - { field: 'outcome.disease_related' }, - { field: 'affected_status' }, + { field: 'participant_id', header: 'Participant ID' }, + { field: 'external_id', header: 'External Participant ID' }, + { field: 'families_id', header: 'Family ID' }, + //TODO { field: '?', header: 'External Family ID' }, + { field: 'is_proband', header: 'Proband' }, + { field: 'study.study_name', header: 'Study Name' }, + { field: 'study.study_code', header: 'Study Code' }, + { field: 'family_type', header: 'Family Composition' }, + //TODO { field: '?', header: 'Diagnosis Category' }, + { field: 'sex', header: 'Sex' }, + { field: 'race', header: 'Race' }, + { field: 'ethnicity', header: 'Ethnicity' }, + { field: 'outcomes.vital_status', header: 'Vital Status' }, + { field: 'outcomes.age_at_event_days.value', header: 'Age at Last Vital Status (Days)' }, ], sort: [ - // does not work: - // see https://www.elastic.co/guide/en/elasticsearch/reference/current/fielddata.html { - family_id: { + participant_id: { order: 'asc', }, }, - { - kf_id: { - order: 'asc', - }, - }, - { - 'diagnoses.age_at_event_days': { - order: 'desc', - }, - }, ], }; @@ -48,147 +32,91 @@ const phenotypes: SheetConfig = { sheetName: 'Phenotypes', root: 'phenotype', columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'is_proband' }, + { field: 'participant_id', header: 'Participant ID' }, + { field: 'external_id', header: 'External Participant ID' }, + { field: 'families_id', header: 'Family ID' }, + //TODO { field: '?', header: 'External Family ID' }, + { field: 'is_proband', header: 'Proband' }, { - field: 'phenotype.observed', + field: 'phenotype.is_observed', additionalFields: ['phenotype.hpo_phenotype_observed', 'phenotype.hpo_phenotype_not_observed'], header: 'Phenotype (HPO)', - transform: (observed, row) => { + transform: ( + isObserved: boolean, + row: { phenotype: { hpo_phenotype_observed: string; hpo_phenotype_not_observed: string } }, + ) => { if (!row.phenotype) { return; } - return observed ? row.phenotype.hpo_phenotype_observed : row.phenotype.hpo_phenotype_not_observed; + return isObserved ? row.phenotype.hpo_phenotype_observed : row.phenotype.hpo_phenotype_not_observed; }, }, + { field: 'phenotype.source_text', header: 'Phenotype (Source Text)' }, { - field: 'phenotype.observed', - additionalFields: ['phenotype.snomed_phenotype_observed', 'phenotype.snomed_phenotype_not_observed'], - header: 'Phenotype (SNOMED)', - transform: (observed, row) => { - if (!row.phenotype) { - return; - } - return observed ? row.phenotype.snomed_phenotype_observed : row.phenotype.snomed_phenotype_not_observed; - }, - }, - { field: 'phenotype.source_text_phenotype' }, - { - field: 'phenotype.observed', + field: 'phenotype.is_observed', header: 'Interpretation', - transform: (value, _row) => (value ? 'Observed' : 'Not Observed'), - }, - { - field: 'phenotype.age_at_event_days', - header: 'Age at Phenotype Assignment (Days)', + transform: (value: boolean) => (value ? 'Observed' : 'Not Observed'), }, + { field: 'phenotype.age_at_event_days', header: 'Age at Phenotype Assignment (Days)' }, ], - sort: [{ kf_id: 'asc' }], + sort: [{ participant_id: 'asc' }], }; -// const diagnoses: SheetConfig = { sheetName: 'Diagnoses', - root: 'diagnoses', + root: 'diagnosis', columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'is_proband' }, - { - field: 'kf_id', - header: 'Diagnosis Type', - transform: () => 'Clinical', - }, - { field: 'diagnoses.diagnosis_category' }, - { field: 'diagnoses.mondo_id_diagnosis' }, - { field: 'diagnoses.ncit_id_diagnosis' }, - { field: 'diagnoses.source_text_diagnosis' }, + { field: 'participant_id', header: 'Participant ID' }, + { field: 'external_id', header: 'External Participant ID' }, + { field: 'families_id', header: 'Family ID' }, + { field: 'is_proband', header: 'Proband' }, + //TODO { field: '?', header: 'Diagnosis Category' }, + + { field: 'diagnosis.mondo_display_term', header: 'Diagnosis (MONDO)' }, + { - field: 'diagnoses.age_at_event_days', - header: 'Age at Diagnosis (Days)', + field: 'diagnosis.ncit_display_term', + additionalFields: ['diagnosis.ncit_code'], + header: 'Diagnosis (NCIT)', + transform: (displayTerm: string, row: { diagnosis: { ncit_code: string } }) => + displayTerm || row?.diagnosis?.ncit_code || '', }, - { field: 'diagnoses.source_text_tumor_location' }, + + { field: 'diagnosis.source_text', header: 'Diagnosis (Source Text)' }, + { field: 'diagnosis.age_at_event_days', header: 'Age at Diagnosis (Days)' }, + { field: 'diagnosis.source_text_tumor_location', header: 'Tumor Location (Source Text)' }, ], - sort: [{ kf_id: 'asc' }], + sort: [{ participant_id: 'asc' }], }; -const histologicalDiagnoses: SheetConfig = { - sheetName: 'Histological Diagnoses', - root: 'biospecimens.diagnoses', +const familyRelationship: SheetConfig = { + sheetName: 'Family Relationship', + root: 'family.relations_to_proband', columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'biospecimens.kf_id' }, - { field: 'biospecimens.external_sample_id' }, - { field: 'biospecimens.external_aliquot_id' }, - { field: 'is_proband' }, { - // This allows to do a cell with a static value. - // The value will be formatted like the value of `field` would have. - field: 'kf_id', - header: 'Diagnosis Type', - transform: () => 'Histological', + field: 'family.relations_to_proband.participant_id', + header: 'Participant ID', + additionalFields: ['participant_id'], + transform: (value: string, row: { participant_id: string }) => value ?? row.participant_id, }, - { field: 'biospecimens.diagnoses.diagnosis_category' }, - { field: 'biospecimens.diagnoses.mondo_id_diagnosis' }, - { field: 'biospecimens.diagnoses.ncit_id_diagnosis' }, - { field: 'biospecimens.diagnoses.source_text_diagnosis' }, { - field: 'biospecimens.diagnoses.age_at_event_days', - header: 'Age at Diagnosis (Days)', + field: 'family.family_id', + header: 'Family ID', }, - { field: 'biospecimens.diagnoses.source_text_tumor_location' }, - { field: 'biospecimens.source_text_anatomical_site' }, - { field: 'biospecimens.ncit_id_tissue_type' }, - { field: 'biospecimens.source_text_tissue_type' }, - { field: 'biospecimens.composition' }, - { field: 'biospecimens.method_of_sample_procurement' }, - { field: 'biospecimens.analyte_type' }, - ], - sort: [ - { kf_id: 'asc' }, - { - 'biospecimens.diagnoses.age_at_event_days': { - order: 'desc', - nested: { - path: 'biospecimens', - nested: { - path: 'biospecimens.diagnoses', - }, - }, - }, - }, - ], -}; - -const familyRelationship: SheetConfig = { - sheetName: 'Family Relationship', - root: 'family.family_compositions.family_members', - columns: [ - { field: 'kf_id' }, - { field: 'family.family_compositions.family_members.kf_id' }, { - field: 'family.family_compositions.family_members.relationship', - header: 'Relationship', - transform: (value) => value || 'self', + field: 'family.relations_to_proband.role', + header: 'Family Role', }, ], - sort: [{ kf_id: 'asc' }], + sort: [{ participant_id: 'asc' }], }; export const queryConfigs: QueryConfig = { indexName: 'participant', - alias: 'participant_centric', + alias: 'next_participant_centric', }; -export const sheetConfigs: SheetConfig[] = [ - participants, - phenotypes, - diagnoses, - histologicalDiagnoses, - familyRelationship, -]; +export const sheetConfigs: SheetConfig[] = [participants, phenotypes, diagnoses, familyRelationship]; const reportConfig: ReportConfig = { queryConfigs, sheetConfigs }; diff --git a/src/reports/clinical-data/configKfNext.ts b/src/reports/clinical-data/configKfNext.ts deleted file mode 100644 index 00c02b6..0000000 --- a/src/reports/clinical-data/configKfNext.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { QueryConfig, ReportConfig, SheetConfig } from '../types'; - -const participants: SheetConfig = { - sheetName: 'Participants', - root: null, - columns: [ - { field: 'participant_id', header: 'Participant ID' }, - { field: 'external_id', header: 'External Participant ID' }, - { field: 'families_id', header: 'Family ID' }, - //TODO { field: '?', header: 'External Family ID' }, - { field: 'is_proband', header: 'Proband' }, - { field: 'study.study_name', header: 'Study Name' }, - { field: 'study.study_code', header: 'Study Code' }, - { field: 'family_type', header: 'Family Composition' }, - //TODO { field: '?', header: 'Diagnosis Category' }, - { field: 'sex', header: 'Sex' }, - { field: 'race', header: 'Race' }, - { field: 'ethnicity', header: 'Ethnicity' }, - { field: 'outcomes.vital_status', header: 'Vital Status' }, - { field: 'outcomes.age_at_event_days.value', header: 'Age at Last Vital Status (Days)' }, - ], - sort: [ - { - participant_id: { - order: 'asc', - }, - }, - ], -}; - -const phenotypes: SheetConfig = { - sheetName: 'Phenotypes', - root: 'phenotype', - columns: [ - { field: 'participant_id', header: 'Participant ID' }, - { field: 'external_id', header: 'External Participant ID' }, - { field: 'families_id', header: 'Family ID' }, - //TODO { field: '?', header: 'External Family ID' }, - { field: 'is_proband', header: 'Proband' }, - { - field: 'phenotype.is_observed', - additionalFields: ['phenotype.hpo_phenotype_observed', 'phenotype.hpo_phenotype_not_observed'], - header: 'Phenotype (HPO)', - transform: ( - isObserved: boolean, - row: { phenotype: { hpo_phenotype_observed: string; hpo_phenotype_not_observed: string } }, - ) => { - if (!row.phenotype) { - return; - } - return isObserved ? row.phenotype.hpo_phenotype_observed : row.phenotype.hpo_phenotype_not_observed; - }, - }, - { field: 'phenotype.source_text', header: 'Phenotype (Source Text)' }, - { - field: 'phenotype.is_observed', - header: 'Interpretation', - transform: (value: boolean) => (value ? 'Observed' : 'Not Observed'), - }, - { field: 'phenotype.age_at_event_days', header: 'Age at Phenotype Assignment (Days)' }, - ], - sort: [{ participant_id: 'asc' }], -}; - -const diagnoses: SheetConfig = { - sheetName: 'Diagnoses', - root: 'diagnosis', - columns: [ - { field: 'participant_id', header: 'Participant ID' }, - { field: 'external_id', header: 'External Participant ID' }, - { field: 'families_id', header: 'Family ID' }, - { field: 'is_proband', header: 'Proband' }, - //TODO { field: '?', header: 'Diagnosis Category' }, - - { field: 'diagnosis.mondo_display_term', header: 'Diagnosis (MONDO)' }, - - { - field: 'diagnosis.ncit_display_term', - additionalFields: ['diagnosis.ncit_code'], - header: 'Diagnosis (NCIT)', - transform: (displayTerm: string, row: { diagnosis: { ncit_code: string } }) => - displayTerm || row?.diagnosis?.ncit_code || '', - }, - - { field: 'diagnosis.source_text', header: 'Diagnosis (Source Text)' }, - { field: 'diagnosis.age_at_event_days', header: 'Age at Diagnosis (Days)' }, - { field: 'diagnosis.source_text_tumor_location', header: 'Tumor Location (Source Text)' }, - ], - sort: [{ participant_id: 'asc' }], -}; - -const familyRelationship: SheetConfig = { - sheetName: 'Family Relationship', - root: 'family.relations_to_proband', - columns: [ - { - field: 'family.relations_to_proband.participant_id', - header: 'Participant ID', - additionalFields: ['participant_id'], - transform: (value: string, row: { participant_id: string }) => value ?? row.participant_id, - }, - { - field: 'family.family_id', - header: 'Family ID', - }, - { - field: 'family.relations_to_proband.role', - header: 'Family Role', - }, - ], - sort: [{ participant_id: 'asc' }], -}; - -export const queryConfigs: QueryConfig = { - indexName: 'participant', - alias: 'next_participant_centric', -}; - -export const sheetConfigs: SheetConfig[] = [participants, phenotypes, diagnoses, familyRelationship]; - -const reportConfig: ReportConfig = { queryConfigs, sheetConfigs }; - -export default reportConfig; diff --git a/src/reports/clinical-data/index.ts b/src/reports/clinical-data/index.ts index 6b87daa..4b5f287 100644 --- a/src/reports/clinical-data/index.ts +++ b/src/reports/clinical-data/index.ts @@ -9,19 +9,16 @@ import generateReport from '../generateReport'; import { ProjectType } from '../types'; import configInclude from './configInclude'; import configKf from './configKf'; -import configKfNext from './configKfNext'; const clinicalDataReport = async (req: Request, res: Response): Promise => { console.time('clinical-data'); - const { sqon, projectId, filename = null, isKfNext = false } = req.body; + const { sqon, projectId, filename = null } = req.body; const userId = req['kauth']?.grant?.access_token?.content?.sub; const accessToken = req.headers.authorization; const p = PROJECT.toLowerCase().trim(); let reportConfig; - if (isKfNext) { - reportConfig = configKfNext; - } else if (p === ProjectType.include) { + if (p === ProjectType.include) { reportConfig = configInclude; } else if (p === ProjectType.kidsFirst) { reportConfig = configKf; diff --git a/src/reports/family-clinical-data/configKf.ts b/src/reports/family-clinical-data/configKf.ts index 79cad1c..4ba3d55 100644 --- a/src/reports/family-clinical-data/configKf.ts +++ b/src/reports/family-clinical-data/configKf.ts @@ -4,42 +4,47 @@ const participants: SheetConfig = { sheetName: 'Participants', root: null, columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'family_id' }, - { field: 'is_proband' }, - { field: 'study.short_name' }, - { field: 'family.family_compositions.composition' }, - { field: 'diagnosis_category' }, - { field: 'gender' }, - { field: 'race' }, - { field: 'ethnicity' }, - { field: 'outcome.vital_status' }, + { field: 'participant_id', header: 'Participant ID' }, + { field: 'external_id', header: 'External Participant ID' }, + { field: 'families_id', header: 'Family ID' }, + //TODO { field: '?', header: 'External Family ID' }, + { field: 'is_proband', header: 'Proband' }, + { field: 'study.study_name', header: 'Study Name' }, + { field: 'study.study_code', header: 'Study Code' }, + { field: 'family_type', header: 'Family Composition' }, { - field: 'outcome.age_at_event_days', - header: 'Age at the Last Vital Status (Days)', + field: 'family.relations_to_proband', + header: 'Family Role', + transform: ( + _: string, + row: { + participant_id: string; + family: { relations_to_proband: { participant_id: string; role: string }[] }; + }, + ): string => { + const ptId = row.participant_id; + const relations = row?.family?.relations_to_proband ?? []; + return relations.find((x) => x.participant_id === ptId)?.role ?? ''; + }, }, - { field: 'outcome.disease_related' }, - { field: 'affected_status' }, + //TODO { field: '?', header: 'Diagnosis Category' }, + { field: 'sex', header: 'Sex' }, + { field: 'race', header: 'Race' }, + { field: 'ethnicity', header: 'Ethnicity' }, + { field: 'outcomes.vital_status', header: 'Vital Status' }, + { field: 'outcomes.age_at_event_days.value', header: 'Age at Last Vital Status (Days)' }, ], sort: [ - // `family.family_id` would not work. - // see https://www.elastic.co/guide/en/elasticsearch/reference/current/fielddata.html { - family_id: { + families_id: { order: 'asc', }, }, { - kf_id: { + participant_id: { order: 'asc', }, }, - { - 'diagnoses.age_at_event_days': { - order: 'desc', - }, - }, ], }; @@ -47,65 +52,63 @@ const phenotypes: SheetConfig = { sheetName: 'Phenotypes', root: 'phenotype', columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'family_id' }, - { field: 'is_proband' }, + { field: 'participant_id', header: 'Participant ID' }, + { field: 'external_id', header: 'External Participant ID' }, + { field: 'families_id', header: 'Family ID' }, + //TODO { field: '?', header: 'External Family ID' }, + { field: 'is_proband', header: 'Proband' }, { - field: 'phenotype.hpo_phenotype_observed_text', + field: 'phenotype.is_observed', + additionalFields: ['phenotype.hpo_phenotype_observed', 'phenotype.hpo_phenotype_not_observed'], header: 'Phenotype (HPO)', - }, - { - field: 'phenotype.observed', - additionalFields: ['phenotype.snomed_phenotype_observed', 'phenotype.snomed_phenotype_not_observed'], - header: 'Phenotype (SNOMED)', - transform: (observed, row) => { + transform: ( + isObserved: boolean, + row: { phenotype: { hpo_phenotype_observed: string; hpo_phenotype_not_observed: string } }, + ) => { if (!row.phenotype) { return; } - return observed ? row.phenotype.snomed_phenotype_observed : row.phenotype.snomed_phenotype_not_observed; + return isObserved ? row.phenotype.hpo_phenotype_observed : row.phenotype.hpo_phenotype_not_observed; }, }, - { field: 'phenotype.source_text_phenotype' }, + { field: 'phenotype.source_text', header: 'Phenotype (Source Text)' }, { - field: 'phenotype.observed', + field: 'phenotype.is_observed', header: 'Interpretation', - transform: (value, _row) => (value ? 'Observed' : 'Not Observed'), + transform: (value: boolean) => (value ? 'Observed' : 'Not Observed'), }, { field: 'phenotype.age_at_event_days', header: 'Age at Phenotype Assignment (Days)' }, ], - sort: [{ family_id: 'asc' }, { kf_id: 'asc' }], + sort: [{ families_id: 'asc' }, { participant_id: 'asc' }], }; const diagnoses: SheetConfig = { sheetName: 'Diagnoses', - root: 'diagnoses', + root: 'diagnosis', columns: [ - { field: 'kf_id' }, - { field: 'external_id' }, - { field: 'family_id' }, - { field: 'is_proband' }, - { - field: 'kf_id', - header: 'Diagnosis Type', - transform: () => 'Clinical', - }, - { field: 'diagnoses.diagnosis_category' }, - { field: 'diagnoses.mondo_id_diagnosis' }, - { field: 'diagnoses.ncit_id_diagnosis' }, - { field: 'diagnoses.source_text_diagnosis' }, + { field: 'participant_id', header: 'Participant ID' }, + { field: 'external_id', header: 'External Participant ID' }, + { field: 'families_id', header: 'Family ID' }, + { field: 'is_proband', header: 'Proband' }, + //TODO { field: '?', header: 'Diagnosis Category' }, + { field: 'diagnosis.mondo_display_term', header: 'Diagnosis (MONDO)' }, { - field: 'diagnoses.age_at_event_days', - header: 'Age at Diagnosis (Days)', + field: 'diagnosis.ncit_display_term', + additionalFields: ['diagnosis.ncit_code'], + header: 'Diagnosis (NCIT)', + transform: (displayTerm: string, row: { diagnosis: { ncit_code: string } }) => + displayTerm || row?.diagnosis?.ncit_code || '', }, - { field: 'diagnoses.source_text_tumor_location' }, + { field: 'diagnosis.source_text', header: 'Diagnosis (Source Text)' }, + { field: 'diagnosis.age_at_event_days', header: 'Age at Diagnosis (Days)' }, + { field: 'diagnosis.source_text_tumor_location', header: 'Tumor Location (Source Text)' }, ], - sort: [{ family_id: 'asc' }, { kf_id: 'asc' }], + sort: [{ families_id: 'asc' }, { participant_id: 'asc' }], }; export const queryConfigs: QueryConfig = { indexName: 'participant', - alias: 'participant_centric', + alias: 'next_participant_centric', }; export const sheetConfigs: SheetConfig[] = [participants, phenotypes, diagnoses]; diff --git a/src/reports/family-clinical-data/configKfNext.ts b/src/reports/family-clinical-data/configKfNext.ts deleted file mode 100644 index 4ba3d55..0000000 --- a/src/reports/family-clinical-data/configKfNext.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { QueryConfig, ReportConfig, SheetConfig } from '../types'; - -const participants: SheetConfig = { - sheetName: 'Participants', - root: null, - columns: [ - { field: 'participant_id', header: 'Participant ID' }, - { field: 'external_id', header: 'External Participant ID' }, - { field: 'families_id', header: 'Family ID' }, - //TODO { field: '?', header: 'External Family ID' }, - { field: 'is_proband', header: 'Proband' }, - { field: 'study.study_name', header: 'Study Name' }, - { field: 'study.study_code', header: 'Study Code' }, - { field: 'family_type', header: 'Family Composition' }, - { - field: 'family.relations_to_proband', - header: 'Family Role', - transform: ( - _: string, - row: { - participant_id: string; - family: { relations_to_proband: { participant_id: string; role: string }[] }; - }, - ): string => { - const ptId = row.participant_id; - const relations = row?.family?.relations_to_proband ?? []; - return relations.find((x) => x.participant_id === ptId)?.role ?? ''; - }, - }, - //TODO { field: '?', header: 'Diagnosis Category' }, - { field: 'sex', header: 'Sex' }, - { field: 'race', header: 'Race' }, - { field: 'ethnicity', header: 'Ethnicity' }, - { field: 'outcomes.vital_status', header: 'Vital Status' }, - { field: 'outcomes.age_at_event_days.value', header: 'Age at Last Vital Status (Days)' }, - ], - sort: [ - { - families_id: { - order: 'asc', - }, - }, - { - participant_id: { - order: 'asc', - }, - }, - ], -}; - -const phenotypes: SheetConfig = { - sheetName: 'Phenotypes', - root: 'phenotype', - columns: [ - { field: 'participant_id', header: 'Participant ID' }, - { field: 'external_id', header: 'External Participant ID' }, - { field: 'families_id', header: 'Family ID' }, - //TODO { field: '?', header: 'External Family ID' }, - { field: 'is_proband', header: 'Proband' }, - { - field: 'phenotype.is_observed', - additionalFields: ['phenotype.hpo_phenotype_observed', 'phenotype.hpo_phenotype_not_observed'], - header: 'Phenotype (HPO)', - transform: ( - isObserved: boolean, - row: { phenotype: { hpo_phenotype_observed: string; hpo_phenotype_not_observed: string } }, - ) => { - if (!row.phenotype) { - return; - } - return isObserved ? row.phenotype.hpo_phenotype_observed : row.phenotype.hpo_phenotype_not_observed; - }, - }, - { field: 'phenotype.source_text', header: 'Phenotype (Source Text)' }, - { - field: 'phenotype.is_observed', - header: 'Interpretation', - transform: (value: boolean) => (value ? 'Observed' : 'Not Observed'), - }, - { field: 'phenotype.age_at_event_days', header: 'Age at Phenotype Assignment (Days)' }, - ], - sort: [{ families_id: 'asc' }, { participant_id: 'asc' }], -}; - -const diagnoses: SheetConfig = { - sheetName: 'Diagnoses', - root: 'diagnosis', - columns: [ - { field: 'participant_id', header: 'Participant ID' }, - { field: 'external_id', header: 'External Participant ID' }, - { field: 'families_id', header: 'Family ID' }, - { field: 'is_proband', header: 'Proband' }, - //TODO { field: '?', header: 'Diagnosis Category' }, - { field: 'diagnosis.mondo_display_term', header: 'Diagnosis (MONDO)' }, - { - field: 'diagnosis.ncit_display_term', - additionalFields: ['diagnosis.ncit_code'], - header: 'Diagnosis (NCIT)', - transform: (displayTerm: string, row: { diagnosis: { ncit_code: string } }) => - displayTerm || row?.diagnosis?.ncit_code || '', - }, - { field: 'diagnosis.source_text', header: 'Diagnosis (Source Text)' }, - { field: 'diagnosis.age_at_event_days', header: 'Age at Diagnosis (Days)' }, - { field: 'diagnosis.source_text_tumor_location', header: 'Tumor Location (Source Text)' }, - ], - sort: [{ families_id: 'asc' }, { participant_id: 'asc' }], -}; - -export const queryConfigs: QueryConfig = { - indexName: 'participant', - alias: 'next_participant_centric', -}; - -export const sheetConfigs: SheetConfig[] = [participants, phenotypes, diagnoses]; - -const reportConfig: ReportConfig = { queryConfigs, sheetConfigs }; - -export default reportConfig; diff --git a/src/reports/family-clinical-data/index.ts b/src/reports/family-clinical-data/index.ts index cc1a7d1..124835f 100644 --- a/src/reports/family-clinical-data/index.ts +++ b/src/reports/family-clinical-data/index.ts @@ -9,21 +9,18 @@ import generateReport from '../generateReport'; import { ProjectType, ReportConfig } from '../types'; import configInclude from './configInclude'; import configKf from './configKf'; -import configKfNext from './configKfNext'; import generatePtSqonWithRelativesIfExist from './generatePtSqonWithRelativesIfExist'; const clinicalDataReport = async (req: Request, res: Response): Promise => { console.time('family-clinical-data'); - const { sqon, projectId, filename = null, isKfNext = false } = req.body; + const { sqon, projectId, filename = null } = req.body; const userId = req['kauth']?.grant?.access_token?.content?.sub; const accessToken = req.headers.authorization; const p = PROJECT.toLowerCase().trim(); let reportConfig: ReportConfig; - if (isKfNext) { - reportConfig = configKfNext; - } else if (p === ProjectType.include) { + if (p === ProjectType.include) { reportConfig = configInclude; } else if (p === ProjectType.kidsFirst) { reportConfig = configKf; diff --git a/src/utils/riffClient.ts b/src/utils/riffClient.ts deleted file mode 100644 index 91d7ded..0000000 --- a/src/utils/riffClient.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { RIFF_URL } from '../env'; -import { RiffError } from './riffError'; -import { Sort, Sqon } from './setsTypes'; - -export type RiffContent = { - setType: string; - riffType: string; - ids: string[]; - sqon: Sqon; - sort: Sort[]; - idField: string; -}; - -export type Riff = { - id: string; - uid: string; - content: RiffContent; - alias: string; - sharedPublicly: boolean; - creationDate: Date; - updatedDate: Date; -}; - -export const getRiffs = async (accessToken: string, userId: string): Promise => { - const uri = `${RIFF_URL}/riff/user/${userId}`; - - const response = await fetch(encodeURI(uri), { - method: 'get', - headers: { - Authorization: accessToken, - 'Content-Type': 'application/json', - }, - }); - - const body = await response.json(); - - if (response.status === 200) { - return body as Riff[]; - } - - throw new RiffError(response.status, body); -}; diff --git a/src/utils/riffError.ts b/src/utils/riffError.ts deleted file mode 100644 index c946b30..0000000 --- a/src/utils/riffError.ts +++ /dev/null @@ -1,12 +0,0 @@ -export class RiffError extends Error { - public readonly status: number; - public readonly details: unknown; - - constructor(status: number, details: unknown) { - super(`Riff returns status ${status}`); - Object.setPrototypeOf(this, RiffError.prototype); - this.name = RiffError.name; - this.status = status; - this.details = details; - } -} diff --git a/src/utils/setsTypes.ts b/src/utils/setsTypes.ts index 7913648..483a0fc 100644 --- a/src/utils/setsTypes.ts +++ b/src/utils/setsTypes.ts @@ -1,5 +1,3 @@ -import { Riff } from './riffClient'; - export type Sqon = { op: string; content: any; // Since SQON is generic, it is too complex to define an explicit type for its content. @@ -28,13 +26,3 @@ export type Output = { creation_date: Date; updated_date: Date; }; - -export const mapRiffOutputToUserOutput = (output: Riff): Output => ({ - id: output.id, - keycloak_id: output.uid, - content: output.content, - alias: output.alias, - sharedpublicly: output.sharedPublicly, - creation_date: output.creationDate, - updated_date: output.updatedDate, -}); diff --git a/src/utils/sqonUtils.test.ts b/src/utils/sqonUtils.test.ts index 535ed94..900abb7 100644 --- a/src/utils/sqonUtils.test.ts +++ b/src/utils/sqonUtils.test.ts @@ -1,229 +1,12 @@ /* eslint-disable @typescript-eslint/no-var-requires */ import { ProjectType } from '../reports/types'; -import { getRiffs, Riff } from './riffClient'; import { Output as UserSetOutput, Sqon } from './setsTypes'; import { resolveSetsInSqon, retrieveSetsFromUsers } from './sqonUtils'; import { getSharedSet, getUserSets } from './userClient'; -jest.mock('./riffClient'); jest.mock('./userClient'); jest.mock('../env'); -describe('resolveSetsInSqon - legacy KF', () => { - beforeEach(() => { - (getRiffs as jest.Mock).mockReset(); - const env = require('../env'); - env.PROJECT = ProjectType.kidsFirst; - }); - - it('should return the input sqon if not set id in it', async () => { - const inputSqonWithoutSetId: Sqon = { - op: 'and', - content: [ - { - op: 'in', - content: { - field: 'gender', - value: ['Female'], - }, - }, - ], - }; - - const result = await resolveSetsInSqon(inputSqonWithoutSetId, 'userId', 'access_token'); - - expect(result).toEqual(inputSqonWithoutSetId); - expect((getRiffs as jest.Mock).mock.calls.length).toEqual(0); - }); - - it('should replace set id by its content', async () => { - const inputSqonWithSetIds: Sqon = { - op: 'and', - content: [ - { - op: 'in', - content: { - field: 'kf_id', - value: ['set_id:1ez'], - }, - }, - { - op: 'in', - content: { - field: 'kf_id', - value: ['set_id:1ey'], - }, - }, - ], - }; - - const riff1ez: Riff = { - id: '1ez', - alias: 'tag-1ez', - content: { - ids: ['participant_1', 'participant_2'], - sqon: { op: 'and', content: [] }, - riffType: 'set', - setType: 'participant', - idField: 'fhir_id', - sort: [], - }, - sharedPublicly: false, - uid: 'uid-1ez', - creationDate: new Date(), - updatedDate: new Date(), - }; - - const riff1ey: Riff = { - id: '1ey', - alias: 'tag-1ey', - content: { - ids: ['participant_1', 'participant_3'], - sqon: { op: 'and', content: [] }, - riffType: 'set', - setType: 'participant', - idField: 'fhir_id', - sort: [], - }, - sharedPublicly: false, - uid: 'uid-1ey', - creationDate: new Date(), - updatedDate: new Date(), - }; - - const anotherRiff: Riff = { - id: '1ea', - alias: 'tag-1ea', - content: { - ids: ['participant_2', 'participant_3'], - sqon: { op: 'and', content: [] }, - riffType: 'set', - setType: 'participant', - idField: 'fhir_id', - sort: [], - }, - sharedPublicly: false, - uid: 'uid-1ea', - creationDate: new Date(), - updatedDate: new Date(), - }; - - const getRiffsMockResponse = [riff1ey, anotherRiff, riff1ez]; - - (getRiffs as jest.Mock).mockImplementation(() => getRiffsMockResponse); - - const expectedResult: Sqon = { - op: 'and', - content: [ - { - op: 'in', - content: { - field: 'kf_id', - value: ['participant_1', 'participant_2'], - }, - }, - { - op: 'in', - content: { - field: 'kf_id', - value: ['participant_1', 'participant_3'], - }, - }, - ], - }; - - const result = await resolveSetsInSqon(inputSqonWithSetIds, 'userId', 'access_token'); - - expect(result).toEqual(expectedResult); - expect((getRiffs as jest.Mock).mock.calls.length).toEqual(1); - }); - - it('should replace set id by an empty array if it has not any set content matching to it', async () => { - const inputSqonWithSetIds: Sqon = { - op: 'and', - content: [ - { - op: 'in', - content: { - field: 'kf_id', - value: ['set_id:1ez'], - }, - }, - { - op: 'in', - content: { - field: 'kf_id', - value: ['set_id:1ex'], - }, - }, - ], - }; - - const riff1ez: Riff = { - id: '1ez', - alias: 'tag-1ez', - content: { - ids: ['participant_1', 'participant_2'], - sqon: { op: 'and', content: [] }, - riffType: 'set', - setType: 'participant', - idField: 'fhir_id', - sort: [], - }, - sharedPublicly: false, - uid: 'uid-1ez', - creationDate: new Date(), - updatedDate: new Date(), - }; - - const anotherRiff: Riff = { - id: '1ea', - alias: 'tag-1ea', - content: { - ids: ['participant_2', 'participant_3'], - sqon: { op: 'and', content: [] }, - riffType: 'set', - setType: 'participant', - idField: 'fhir_id', - sort: [], - }, - sharedPublicly: false, - uid: 'uid-1ea', - creationDate: new Date(), - updatedDate: new Date(), - }; - - const getRiffsMockResponse = [anotherRiff, riff1ez]; - - (getRiffs as jest.Mock).mockImplementation(() => getRiffsMockResponse); - - const expectedResult: Sqon = { - op: 'and', - content: [ - { - op: 'in', - content: { - field: 'kf_id', - value: ['participant_1', 'participant_2'], - }, - }, - { - op: 'in', - content: { - field: 'kf_id', - value: [], - }, - }, - ], - }; - - const result = await resolveSetsInSqon(inputSqonWithSetIds, 'userId', 'access_token'); - - expect(result).toEqual(expectedResult); - expect((getRiffs as jest.Mock).mock.calls.length).toEqual(1); - }); -}); - describe('retrieveSetsFromUsers', () => { beforeEach(() => { (getUserSets as jest.Mock).mockReset(); diff --git a/src/utils/sqonUtils.ts b/src/utils/sqonUtils.ts index 6222e79..d76f4d0 100644 --- a/src/utils/sqonUtils.ts +++ b/src/utils/sqonUtils.ts @@ -1,21 +1,12 @@ import { Dictionary, flattenDeep, get, isArray, zipObject } from 'lodash'; -import { PROJECT } from '../env'; -import { ProjectType } from '../reports/types'; -import { getRiffs } from './riffClient'; -import { mapRiffOutputToUserOutput, Output as UserSetOutput, Sqon } from './setsTypes'; +import { Output as UserSetOutput, Sqon } from './setsTypes'; import { getSharedSet, getUserSets } from './userClient'; export const resolveSetsInSqon = async (sqon: Sqon, userId: string, accessToken: string): Promise => { const setIds: string[] = getSetIdsFromSqon(sqon || ({} as Sqon)); if (setIds.length) { - let userSets: UserSetOutput[]; - const p = PROJECT.toLowerCase().trim(); - if (p === ProjectType.include) { - userSets = await retrieveSetsFromUsers(accessToken, setIds); - } else { - userSets = (await getRiffs(accessToken, userId)).map((s) => mapRiffOutputToUserOutput(s)); - } + const userSets: UserSetOutput[] = await retrieveSetsFromUsers(accessToken, setIds); const ids = setIds.map((setId) => get(userSets.filter((r) => r.id === setId)[0], 'content.ids', [])); const setIdsToValueMap: Dictionary = zipObject(