From e3ab5da201d07d14ea7eb2ab71ba2e293014fc57 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Thu, 15 Sep 2022 16:53:34 +0200 Subject: [PATCH 01/17] Create tongaRawData custom report --- packages/report-server/package.json | 1 + .../customReports/data/tongaCovidRawData.json | 82 +++++ .../data/tongaCovidRawDataFaster.json | 38 +++ .../src/reportBuilder/customReports/index.ts | 8 +- .../customReports/tongaCovidRawData.ts | 305 ++++++++++++++++++ .../customReports/tongaCovidRawDataFaster.ts | 135 ++++++++ yarn.lock | 8 + 7 files changed, 576 insertions(+), 1 deletion(-) create mode 100644 packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json create mode 100644 packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json create mode 100644 packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts create mode 100644 packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts diff --git a/packages/report-server/package.json b/packages/report-server/package.json index 5ef33ea455..76c1ead6ea 100644 --- a/packages/report-server/package.json +++ b/packages/report-server/package.json @@ -39,6 +39,7 @@ "api-error-handler": "^1.0.0", "body-parser": "^1.18.3", "cors": "^2.8.5", + "date-fns": "^2.29.3", "dotenv": "^8.2.0", "express": "^4.16.2", "lodash.isplainobject": "^4.0.6", diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json new file mode 100644 index 0000000000..a456d570a4 --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json @@ -0,0 +1,82 @@ +{ + "C19T_Registration": { + "dataElementCodes": ["C19T002","C19T003","C19T004","C19T005","C19T006"] + }, + "C19T_Results": { + "dataElementCodes": ["C19T012","C19T013","C19T013_a","C19T015","C19T015_a","C19T016","C19T042","C19T017","C19T018","C19T019","C19T021","C19T022","C19T020","C19T038","C19T039","C19T044","C19T024","C19T025","C19T026","C19T027","C19T028","C19T041","C19T029","C19T033","C19T034","C19T035","C19T036","C19T037","C19T043"] + }, + "codesToNames": { + "C19T002": "Surname", + "C19T003": "Given Names", + "C19T004": "dob", + "C19T005": "Sex", + "C19T006": "Phone No.", + "C19T012": "Test Type", + "C19T013": "Other Type", + "C19T013_a":"CT Value", + "C19T015": "Outbound Traveller", + "C19T015_a": "Inbound Traveller", + "C19T016": "Symptomatic", + "C19T042": "Date of Symptom Onset", + "C19T017": "Quarantine", + "C19T018": "Primary Contact", + "C19T019": "Frontliner", + "C19T021": "Frontliner Type", + "C19T022": "Other", + "C19T020": "Community Testing", + "C19T038": "Patient", + "C19T039": "Other Reason", + "C19T044": "Primary Contact Testing Day", + "C19T024": "Testing Site", + "C19T025": "Quarantine Facility", + "C19T026": "Ward Type", + "C19T027": "Clinic Type", + "C19T028": "Health Center", + "C19T041": "RRT", + "C19T029": "Other Site", + "C19T033": "Result", + "C19T034": "Other Results", + "C19T035": "New Case", + "C19T036": "Previous Positive", + "C19T037": "Date Previous Positive", + "C19T043": "Vaccination Status" + }, + "columns": [ + "Test ID", + "Surname", + "Given Names", + "Date of Birth", + "Age", + "Sex", + "Phone No.", + "Island Group", + "Village Code", + "Address", + "Date of Test", + "Result", + "New Case", + "Estimated Recovery Date", + "Test Type", + "CT Value", + "Vaccination Status", + "Outbound Traveller", + "inbound Traveller", + "Symptomatic", + "Date of Symptomatic Onset", + "Quarantine", + "Primary Contact", + "Frontliner", + "Frontliner Type", + "Other", + "Community Testing", + "Patient", + "Other Reason", + "Primary Contact Testing Day", + "Testing Site", + "Quarantine Facility", + "Ward Type", + "Clinic Type", + "Health Center", + "RRT Team Name" + ] +} \ No newline at end of file diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json new file mode 100644 index 0000000000..09c7b49dd2 --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json @@ -0,0 +1,38 @@ +[ + "Test ID", + "Surname", + "Given Names", + "Date of Birth", + "Age", + "Sex", + "Phone No.", + "Island Group", + "Village Code", + "Address", + "Date of Test", + "Result", + "New Case", + "Estimated Recovery Date", + "Test Type", + "CT Value", + "Vaccination Status", + "Outbound Traveller", + "inbound Traveller", + "Symptomatic", + "Date of Symptomatic Onset", + "Quarantine", + "Primary Contact", + "Frontliner", + "Frontliner Type", + "Other", + "Community Testing", + "Patient", + "Other Reason", + "Primary Contact Testing Day", + "Testing Site", + "Quarantine Facility", + "Ward Type", + "Clinic Type", + "Health Center", + "RRT Team Name" +] \ No newline at end of file diff --git a/packages/report-server/src/reportBuilder/customReports/index.ts b/packages/report-server/src/reportBuilder/customReports/index.ts index cba577a298..07fead265c 100644 --- a/packages/report-server/src/reportBuilder/customReports/index.ts +++ b/packages/report-server/src/reportBuilder/customReports/index.ts @@ -7,10 +7,16 @@ import { Resolved } from '@tupaia/tsutils'; import { FetchReportQuery } from '../../types'; import { ReqContext } from '../context'; import { testCustomReport } from './testCustomReport'; +import { tongaCovidRawData } from './tongaCovidRawData'; +import { tongaCovidRawDataFaster } from './tongaCovidRawDataFaster'; type CustomReportBuilder = (reqContext: ReqContext, query: FetchReportQuery) => Promise; -export const customReports: Record = { testCustomReport }; +export const customReports: Record = { + testCustomReport, + tongaCovidRawData, + tongaCovidRawDataFaster, +}; export type CustomReportOutputType = Resolved< ReturnType diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts new file mode 100644 index 0000000000..5fd3aa380e --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -0,0 +1,305 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ +import { format, differenceInYears, addDays, isDate } from 'date-fns'; +import { createAggregator } from '@tupaia/aggregator'; +import { ReportServerAggregator } from '../../aggregator'; +import { FetchReportQuery, Event } from '../../types'; +import { ReqContext } from '../context'; +import SURVEYS from './data/tongaCovidRawData.json'; + +interface RelationshipsOptions { + hierarchy: string; + individualsByCode: string[]; + groupBy: 'ancestor' | 'descendant'; + queryOptions?: any; + ancestorOptions?: any; + descendantOptions?: any; +} + +interface Options { + programCode: string; + dataElementCodes: string[]; +} + +interface AncestorData { + villageByIndividual: Record>; + islandByIndividual: Record; +} + +const NOW = new Date(); + +const getRelationships = (reqContext: ReqContext, options: RelationshipsOptions) => { + const { + hierarchy, + individualsByCode, + groupBy, + queryOptions, + ancestorOptions, + descendantOptions, + } = options; + return reqContext.services.entity.getRelationshipsOfEntities( + hierarchy, + individualsByCode, + groupBy, + queryOptions, + ancestorOptions, + descendantOptions, + ); +}; + +const useAncestorData = async ( + reqContext: ReqContext, + hierarchy: string, + individualsByCode: string[], +) => { + const villageCodeByIndividualCodes: Record = await getRelationships(reqContext, { + hierarchy, + individualsByCode, + groupBy: 'descendant', + queryOptions: {}, + ancestorOptions: { filter: { type: 'village' } }, + descendantOptions: { filter: { type: 'individual' } }, + }); + const villageCodes = new Set(); + Object.values(villageCodeByIndividualCodes).forEach((code: string) => { + villageCodes.add(code); + }); + const includedVillageCodes = [...villageCodes]; + const villageCodesAndNames = await reqContext.services.entity.getEntities( + hierarchy, + includedVillageCodes, + { + fields: ['code', 'name'], + }, + ); + const individualCodeByVillageNameAndCode: Record> = {}; + Object.keys(villageCodeByIndividualCodes).forEach(individualCode => { + const villageCode = villageCodeByIndividualCodes[individualCode]; + const { name } = villageCodesAndNames.find( + (village: Record) => village.code === villageCode, + ); + individualCodeByVillageNameAndCode[individualCode] = { + code: villageCode, + name, + }; + }); + + const islandNameByIndividualCodes = await getRelationships(reqContext, { + hierarchy, + individualsByCode, + groupBy: 'descendant', + queryOptions: { field: 'name' }, + ancestorOptions: { filter: { type: 'district' } }, + descendantOptions: { filter: { type: 'individual' } }, + }); + + return { + villageByIndividual: individualCodeByVillageNameAndCode, + islandByIndividual: islandNameByIndividualCodes, + }; +}; + +const fetchEvents = async ( + reqContext: ReqContext, + individualsByCode: string[], + hierarchy: string, + startDate?: string, + endDate?: string, + period?: string, +) => { + const registrationOptions = { + programCode: 'C19T_Registration', + dataElementCodes: SURVEYS.C19T_Registration.dataElementCodes, + }; + const resultsOptions = { + programCode: 'C19T_Results', + dataElementCodes: SURVEYS.C19T_Results.dataElementCodes, + }; + + const aggregator = new ReportServerAggregator(createAggregator(undefined, reqContext)); + const fetch = async (options: Options) => { + const { programCode, dataElementCodes } = options; + return aggregator.fetchEvents( + programCode, + undefined, + individualsByCode, + hierarchy, + { startDate, endDate, period }, + dataElementCodes, + ); + }; + + const registrationEvents = await fetch(registrationOptions); + const resultsEvents = await fetch(resultsOptions); + return { registrationEvents, resultsEvents }; +}; + +const combineAndFlatten = ( + registrationEvents: Event[], + resultEvents: Event[], + ancestorData: AncestorData, +) => { + const matchedData: Record[] = resultEvents.map(resultEvent => { + const { dataValues: resultDataValues, orgUnit: resultOrgUnit, eventDate } = resultEvent; + const matchingRegistration = registrationEvents.find( + registrationEvent => registrationEvent.orgUnit === resultEvent.orgUnit, + ); + if (!matchingRegistration) { + return { + orgUnit: resultOrgUnit, + eventDate, + ...resultDataValues, + }; + } + const { dataValues: registrationDataValues, orgUnit } = matchingRegistration; + + return { + orgUnit, + eventDate, + ...registrationDataValues, + ...resultDataValues, + }; + }); + const { villageByIndividual, islandByIndividual } = ancestorData; + + const dataWithUpdatesAndAddOns = matchedData.map(event => { + const { + orgUnit, + C19T033: result, + C19T042: onsetDate, + C19T004: dob, + eventDate: dateSpecimenCollected, + } = event; + const { code: villageCode, name: villageName } = villageByIndividual[ + orgUnit as keyof typeof Event + ]; + const islandName: string = islandByIndividual[orgUnit]; + const age = getAge(dob); + return { + 'Test ID': orgUnit, + 'Island Group': islandName, + 'Village Code': villageCode, + Address: villageName, + 'Estimated Recovery Date': getEstimatedRecoveryDate(result, dateSpecimenCollected, onsetDate), + dateSpecimenCollected, + Age: age, + ...event, + }; + }); + + return dataWithUpdatesAndAddOns; +}; + +const getAge = (dob: string | undefined) => { + if (!dob) { + return 'unknown'; + } + const dobDate = new Date(dob); + const age = isDate(dobDate) && differenceInYears(NOW, dobDate); + return age; +}; + +const getEstimatedRecoveryDate = ( + result: string, + collectionDate: string, + onsetDate: string | undefined, +) => { + if (result !== 'Positive') { + return 'Not applicable'; + } + + if (onsetDate) { + const recoveryDate = addDays(new Date(onsetDate), 13); + + return isDate(recoveryDate) && format(recoveryDate, 'yyy-mm-dd'); + } + + const recoveryDate = addDays(new Date(collectionDate), 13); + return isDate(recoveryDate) && format(recoveryDate, 'yyyy-mm-dd'); +}; + +const parseRowData = (rowData: Record) => { + const formattedRow: Record = {}; + const { codesToNames } = SURVEYS; + Object.keys(rowData).forEach(fieldKey => { + switch (fieldKey) { + case 'dateSpecimenCollected': { + formattedRow['Date of Test'] = rowData[fieldKey]; + break; + } + case 'C19T004': { + if (!rowData[fieldKey]) { + formattedRow['Date of Birth'] = 'Unknown'; + } + const rawDate = new Date(rowData[fieldKey]); + const dobValue = isDate(rawDate) && format(rawDate, 'yyyy-mm-dd'); + formattedRow['Date of Birth'] = dobValue; + break; + } + case 'Test ID': + case 'Island Group': + case 'Village Code': + case 'Address': + case 'Age': + case 'Estimated Recovery Date': + formattedRow[fieldKey] = rowData[fieldKey]; + break; + default: { + const name = codesToNames[fieldKey as keyof typeof codesToNames]; + if (!name) { + formattedRow[fieldKey] = rowData[fieldKey]; + } else { + formattedRow[name] = rowData[fieldKey]; + } + } + } + }); + return formattedRow; +}; + +export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchReportQuery) => { + const { organisationUnitCodes: entityCodes, hierarchy, startDate, endDate, period } = query; + const individuals = await reqContext.services.entity.getDescendantsOfEntities( + hierarchy, + entityCodes, + { filter: { type: 'individual' } }, + ); + const individualsByCode = individuals.map((ind: Record) => ind.code); + + const ancestorData: AncestorData = await useAncestorData( + reqContext, + hierarchy, + individualsByCode, + ); + + const { registrationEvents, resultsEvents } = await fetchEvents( + reqContext, + individualsByCode, + hierarchy, + startDate, + endDate, + period, + ); + + const builtEvents: Record[] = combineAndFlatten( + registrationEvents, + resultsEvents, + ancestorData, + ); + const rows = builtEvents + .map(rowData => parseRowData(rowData)) + .sort((row, nextRow) => { + if (row['Test ID'] === nextRow['Test ID']) { + return 0; + } + return 1; + }); + + const columns: Record[] = SURVEYS.columns.map(key => { + return { title: key, key }; + }); + + return { columns, rows }; +}; diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts new file mode 100644 index 0000000000..547a937edb --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts @@ -0,0 +1,135 @@ +import { TupaiaDatabase } from '@tupaia/database'; +import { string } from 'mathjs'; +import KEYS from './data/tongaCovidRawDataFaster.json'; + +export const tongaCovidRawDataFaster = async () => { + const db = new TupaiaDatabase(); + const rows: Record[] = await db.executeSql( + ` + select + C19.lab_id as "Test ID", + C19.surname as "Surname", + C19.first_name as "Given Names", + C19.dob as "Date of Birth", + C19.age as "Age", + C19.sex as "Sex", + C19.phone as "Phone No.", + C19.island as "Island Group", + C19.village_code as "Village Code", + C19.village as "Address", + C19.date_specimen_collected as "Date of Test", + C19.results as "Result", + C19.new_case as "New Case", + (case + when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is not null then (C19.date_symptom_onset::date + interval '13' day)::date + when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is null then (C19.date_specimen_collected + interval '13' day)::date + when C19.results = 'Positive' then (C19.date_specimen_collected + interval '10' day)::date + end) as "Estimated Recovery Date", + C19.test_type as "Test Type", + C19.ctvalue as "CT Value", + C19.vax_status as "Vaccination Status", + C19.outbound_traveller as "Outbound Traveller", + C19.inbound_traveller as "Inbound Traveller", + C19.symptomatic as "Symptomatic", + C19.date_symptom_onset as "Date of Symptomatic Onset", + C19.quarantine as "Quarantine", + C19.linkedtocase as "Primary Contact", + C19.frontliner as "Frontliner", + C19.frontliner_type as "Frontliner Type", + C19.other_frontliner_type as "Other", + C19.communitytesting as "Community Testing", + C19.patient as "Patient", + C19.other_reason as "Other Reason", + C19.primarycontact_testingday as "Primary Contact Testing Day", + C19.specimen_site as "Testing Site", + C19.quarantine_facility as "Quarantine Facility", + C19.ward_type as "Ward Type", + C19.clinic_type as "Clinic Type", + hc.name as "Health Center", + C19.rrt_team as "RRT Team Name", + C19.other_site as "Other" + from + ( + select + e.code as lab_id, + reg.date as date_registered, + reg.surname, + reg.first_name, + reg.dob, + date_part('year', age(sr.data_time::date, reg.dob::date)) as age, + reg.sex, + reg.phone, + ggp.name as island, + p.code as village_code, + p.name as village, + sr.data_time as date_specimen_collected, + max(case when q.code = 'C19T012' then a.text end) as test_type, + max(case when q.code = 'C19T013' then a.text end) as other_type, + max(case when q.code = 'C19T013_a' then a.text end) as ctvalue, + max(case when q.code = 'C19T015' then a.text end) as outbound_traveller, + max(case when q.code = 'C19T015_a' then a.text end) as inbound_traveller, + max(case when q.code = 'C19T016' then a.text end) as symptomatic, + max(case when q.code = 'C19T042' then a.text end) as date_symptom_onset, + max(case when q.code = 'C19T017' then a.text end) as quarantine, + max(case when q.code = 'C19T018' then a.text end) as linkedtocase, + max(case when q.code = 'C19T019' then a.text end) as frontliner, + max(case when q.code = 'C19T021' then a.text end) as frontliner_type, + max(case when q.code = 'C19T022' then a.text end) as other_frontliner_type, + max(case when q.code = 'C19T020' then a.text end) as communitytesting, + max(case when q.code = 'C19T038' then a.text end) as patient, + max(case when q.code = 'C19T039' then a.text end) as other_reason, + max(case when q.code = 'C19T044' then a.text end) as primarycontact_testingday, + max(case when q.code = 'C19T024' then a.text end) as specimen_site, + max(case when q.code = 'C19T025' then a.text end) as quarantine_facility, + max(case when q.code = 'C19T026' then a.text end) as ward_type, + max(case when q.code = 'C19T027' then a.text end) as clinic_type, + max(case when q.code = 'C19T028' then a.text end) as health_centre, + max(case when q.code = 'C19T041' then a.text end) as rrt_team, + max(case when q.code = 'C19T029' then a.text end) as other_site, + max(case when q.code = 'C19T032' then a.text end) as rat, + max(case when q.code = 'C19T033' then a.text end) as results, + max(case when q.code = 'C19T034' then a.text end) as other_results, + max(case when q.code = 'C19T035' then a.text end) as new_case, + max(case when q.code = 'C19T036' then a.text end) as previous_positive, + max(case when q.code = 'C19T037' then to_char(a.text::timestamp::date, 'yyyy-mm-dd') end) as date_previous_positive, + max(case when q.code = 'C19T043' then a.text end) as vax_status + from survey_response sr + join answer a on a.survey_response_id = sr.id + join question q on q.id = a.question_id + join survey s on s.id = sr.survey_id + join entity e on e.id = sr.entity_id + join entity p on p.id = e.parent_id + join entity gp on gp.id = p.parent_id + join entity ggp on ggp.id = gp.parent_id + join + ( + select + e1.code as entity_code, + sr1.data_time as date, + max(case when q1.code = 'C19T002' then a1.text end) as surname, + max(case when q1.code = 'C19T003' then a1.text end) as first_name, + max(case when q1.code = 'C19T004' then to_char(a1.text::timestamp::date, 'yyyy-mm-dd') end) as dob, + max(case when q1.code = 'C19T005' then a1.text end) as sex, + max(case when q1.code = 'C19T006' then a1.text end) as phone + from survey_response sr1 + join answer a1 on a1.survey_response_id = sr1.id + join question q1 on q1.id = a1.question_id + join survey s1 on s1.id = sr1.survey_id + join entity e1 on e1.id = sr1.entity_id + where s1.code = 'C19T_Registration' and e1.country_code = 'TO' + group by e1.code, sr1.data_time + ) reg on reg.entity_code = e.code + where s.code = 'C19T_Results' and e.country_code = 'TO' + group by sr.id, reg.date, reg.surname, reg.first_name, reg.dob, reg.sex, reg.phone, sr.data_time, e.code, p.name, p.code, ggp.name + ) C19 + left join entity hc on hc.id = C19.health_centre + `, + [], + ); + + const columns: Record[] = KEYS.map(key => { + return { title: key, key }; + }); + + return { columns, rows: rows.filter((row, index) => index < 20) }; +}; diff --git a/yarn.lock b/yarn.lock index c3dc13d627..141a041132 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5580,6 +5580,7 @@ __metadata: api-error-handler: ^1.0.0 body-parser: ^1.18.3 cors: ^2.8.5 + date-fns: ^2.29.3 dotenv: ^8.2.0 express: ^4.16.2 lodash.isplainobject: ^4.0.6 @@ -12641,6 +12642,13 @@ __metadata: languageName: node linkType: hard +"date-fns@npm:^2.29.3": + version: 2.29.3 + resolution: "date-fns@npm:2.29.3" + checksum: e01cf5b62af04e05dfff921bb9c9933310ed0e1ae9a81eb8653452e64dc841acf7f6e01e1a5ae5644d0337e9a7f936175fd2cb6819dc122fdd9c5e86c56be484 + languageName: node + linkType: hard + "dayjs@npm:^1.8.15": version: 1.9.6 resolution: "dayjs@npm:1.9.6" From 968fb43cbeaff71944aa50eeedc661638f5ac1aa Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Fri, 16 Sep 2022 08:41:55 +0200 Subject: [PATCH 02/17] Run fetches through promise --- .../customReports/tongaCovidRawData.ts | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts index 5fd3aa380e..9fb2321753 100644 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -28,8 +28,6 @@ interface AncestorData { islandByIndividual: Record; } -const NOW = new Date(); - const getRelationships = (reqContext: ReqContext, options: RelationshipsOptions) => { const { hierarchy, @@ -54,14 +52,27 @@ const useAncestorData = async ( hierarchy: string, individualsByCode: string[], ) => { - const villageCodeByIndividualCodes: Record = await getRelationships(reqContext, { + const villageOptions: RelationshipsOptions = { hierarchy, individualsByCode, groupBy: 'descendant', queryOptions: {}, ancestorOptions: { filter: { type: 'village' } }, descendantOptions: { filter: { type: 'individual' } }, - }); + }; + + const islandOptions: RelationshipsOptions = { + hierarchy, + individualsByCode, + groupBy: 'descendant', + queryOptions: { field: 'name' }, + ancestorOptions: { filter: { type: 'district' } }, + descendantOptions: { filter: { type: 'individual' } }, + }; + const villageCodeByIndividualCodes: Record = await getRelationships( + reqContext, + villageOptions, + ); const villageCodes = new Set(); Object.values(villageCodeByIndividualCodes).forEach((code: string) => { villageCodes.add(code); @@ -86,14 +97,7 @@ const useAncestorData = async ( }; }); - const islandNameByIndividualCodes = await getRelationships(reqContext, { - hierarchy, - individualsByCode, - groupBy: 'descendant', - queryOptions: { field: 'name' }, - ancestorOptions: { filter: { type: 'district' } }, - descendantOptions: { filter: { type: 'individual' } }, - }); + const islandNameByIndividualCodes = await getRelationships(reqContext, islandOptions); return { villageByIndividual: individualCodeByVillageNameAndCode, @@ -131,8 +135,10 @@ const fetchEvents = async ( ); }; - const registrationEvents = await fetch(registrationOptions); - const resultsEvents = await fetch(resultsOptions); + const [registrationEvents, resultsEvents] = await Promise.all( + [registrationOptions, resultsOptions].map(fetch), + ); + return { registrationEvents, resultsEvents }; }; @@ -163,7 +169,7 @@ const combineAndFlatten = ( }; }); const { villageByIndividual, islandByIndividual } = ancestorData; - + const now = new Date(); const dataWithUpdatesAndAddOns = matchedData.map(event => { const { orgUnit, @@ -176,7 +182,8 @@ const combineAndFlatten = ( orgUnit as keyof typeof Event ]; const islandName: string = islandByIndividual[orgUnit]; - const age = getAge(dob); + + const age = getAge(dob, now); return { 'Test ID': orgUnit, 'Island Group': islandName, @@ -192,12 +199,12 @@ const combineAndFlatten = ( return dataWithUpdatesAndAddOns; }; -const getAge = (dob: string | undefined) => { +const getAge = (dob: string | undefined, now: Date) => { if (!dob) { return 'unknown'; } const dobDate = new Date(dob); - const age = isDate(dobDate) && differenceInYears(NOW, dobDate); + const age = isDate(dobDate) && differenceInYears(now, dobDate); return age; }; @@ -301,5 +308,5 @@ export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchRepo return { title: key, key }; }); - return { columns, rows }; + return { columns, rows: rows.filter((row, index) => index < 20) }; }; From 605614e18e4333c4e3c347873940fbcdb1bda51d Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Fri, 16 Sep 2022 11:51:36 +0200 Subject: [PATCH 03/17] Delete SQL version --- .../data/tongaCovidRawDataFaster.json | 38 ----- .../customReports/tongaCovidRawDataFaster.ts | 135 ------------------ 2 files changed, 173 deletions(-) delete mode 100644 packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json delete mode 100644 packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json deleted file mode 100644 index 09c7b49dd2..0000000000 --- a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - "Test ID", - "Surname", - "Given Names", - "Date of Birth", - "Age", - "Sex", - "Phone No.", - "Island Group", - "Village Code", - "Address", - "Date of Test", - "Result", - "New Case", - "Estimated Recovery Date", - "Test Type", - "CT Value", - "Vaccination Status", - "Outbound Traveller", - "inbound Traveller", - "Symptomatic", - "Date of Symptomatic Onset", - "Quarantine", - "Primary Contact", - "Frontliner", - "Frontliner Type", - "Other", - "Community Testing", - "Patient", - "Other Reason", - "Primary Contact Testing Day", - "Testing Site", - "Quarantine Facility", - "Ward Type", - "Clinic Type", - "Health Center", - "RRT Team Name" -] \ No newline at end of file diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts deleted file mode 100644 index 547a937edb..0000000000 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { TupaiaDatabase } from '@tupaia/database'; -import { string } from 'mathjs'; -import KEYS from './data/tongaCovidRawDataFaster.json'; - -export const tongaCovidRawDataFaster = async () => { - const db = new TupaiaDatabase(); - const rows: Record[] = await db.executeSql( - ` - select - C19.lab_id as "Test ID", - C19.surname as "Surname", - C19.first_name as "Given Names", - C19.dob as "Date of Birth", - C19.age as "Age", - C19.sex as "Sex", - C19.phone as "Phone No.", - C19.island as "Island Group", - C19.village_code as "Village Code", - C19.village as "Address", - C19.date_specimen_collected as "Date of Test", - C19.results as "Result", - C19.new_case as "New Case", - (case - when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is not null then (C19.date_symptom_onset::date + interval '13' day)::date - when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is null then (C19.date_specimen_collected + interval '13' day)::date - when C19.results = 'Positive' then (C19.date_specimen_collected + interval '10' day)::date - end) as "Estimated Recovery Date", - C19.test_type as "Test Type", - C19.ctvalue as "CT Value", - C19.vax_status as "Vaccination Status", - C19.outbound_traveller as "Outbound Traveller", - C19.inbound_traveller as "Inbound Traveller", - C19.symptomatic as "Symptomatic", - C19.date_symptom_onset as "Date of Symptomatic Onset", - C19.quarantine as "Quarantine", - C19.linkedtocase as "Primary Contact", - C19.frontliner as "Frontliner", - C19.frontliner_type as "Frontliner Type", - C19.other_frontliner_type as "Other", - C19.communitytesting as "Community Testing", - C19.patient as "Patient", - C19.other_reason as "Other Reason", - C19.primarycontact_testingday as "Primary Contact Testing Day", - C19.specimen_site as "Testing Site", - C19.quarantine_facility as "Quarantine Facility", - C19.ward_type as "Ward Type", - C19.clinic_type as "Clinic Type", - hc.name as "Health Center", - C19.rrt_team as "RRT Team Name", - C19.other_site as "Other" - from - ( - select - e.code as lab_id, - reg.date as date_registered, - reg.surname, - reg.first_name, - reg.dob, - date_part('year', age(sr.data_time::date, reg.dob::date)) as age, - reg.sex, - reg.phone, - ggp.name as island, - p.code as village_code, - p.name as village, - sr.data_time as date_specimen_collected, - max(case when q.code = 'C19T012' then a.text end) as test_type, - max(case when q.code = 'C19T013' then a.text end) as other_type, - max(case when q.code = 'C19T013_a' then a.text end) as ctvalue, - max(case when q.code = 'C19T015' then a.text end) as outbound_traveller, - max(case when q.code = 'C19T015_a' then a.text end) as inbound_traveller, - max(case when q.code = 'C19T016' then a.text end) as symptomatic, - max(case when q.code = 'C19T042' then a.text end) as date_symptom_onset, - max(case when q.code = 'C19T017' then a.text end) as quarantine, - max(case when q.code = 'C19T018' then a.text end) as linkedtocase, - max(case when q.code = 'C19T019' then a.text end) as frontliner, - max(case when q.code = 'C19T021' then a.text end) as frontliner_type, - max(case when q.code = 'C19T022' then a.text end) as other_frontliner_type, - max(case when q.code = 'C19T020' then a.text end) as communitytesting, - max(case when q.code = 'C19T038' then a.text end) as patient, - max(case when q.code = 'C19T039' then a.text end) as other_reason, - max(case when q.code = 'C19T044' then a.text end) as primarycontact_testingday, - max(case when q.code = 'C19T024' then a.text end) as specimen_site, - max(case when q.code = 'C19T025' then a.text end) as quarantine_facility, - max(case when q.code = 'C19T026' then a.text end) as ward_type, - max(case when q.code = 'C19T027' then a.text end) as clinic_type, - max(case when q.code = 'C19T028' then a.text end) as health_centre, - max(case when q.code = 'C19T041' then a.text end) as rrt_team, - max(case when q.code = 'C19T029' then a.text end) as other_site, - max(case when q.code = 'C19T032' then a.text end) as rat, - max(case when q.code = 'C19T033' then a.text end) as results, - max(case when q.code = 'C19T034' then a.text end) as other_results, - max(case when q.code = 'C19T035' then a.text end) as new_case, - max(case when q.code = 'C19T036' then a.text end) as previous_positive, - max(case when q.code = 'C19T037' then to_char(a.text::timestamp::date, 'yyyy-mm-dd') end) as date_previous_positive, - max(case when q.code = 'C19T043' then a.text end) as vax_status - from survey_response sr - join answer a on a.survey_response_id = sr.id - join question q on q.id = a.question_id - join survey s on s.id = sr.survey_id - join entity e on e.id = sr.entity_id - join entity p on p.id = e.parent_id - join entity gp on gp.id = p.parent_id - join entity ggp on ggp.id = gp.parent_id - join - ( - select - e1.code as entity_code, - sr1.data_time as date, - max(case when q1.code = 'C19T002' then a1.text end) as surname, - max(case when q1.code = 'C19T003' then a1.text end) as first_name, - max(case when q1.code = 'C19T004' then to_char(a1.text::timestamp::date, 'yyyy-mm-dd') end) as dob, - max(case when q1.code = 'C19T005' then a1.text end) as sex, - max(case when q1.code = 'C19T006' then a1.text end) as phone - from survey_response sr1 - join answer a1 on a1.survey_response_id = sr1.id - join question q1 on q1.id = a1.question_id - join survey s1 on s1.id = sr1.survey_id - join entity e1 on e1.id = sr1.entity_id - where s1.code = 'C19T_Registration' and e1.country_code = 'TO' - group by e1.code, sr1.data_time - ) reg on reg.entity_code = e.code - where s.code = 'C19T_Results' and e.country_code = 'TO' - group by sr.id, reg.date, reg.surname, reg.first_name, reg.dob, reg.sex, reg.phone, sr.data_time, e.code, p.name, p.code, ggp.name - ) C19 - left join entity hc on hc.id = C19.health_centre - `, - [], - ); - - const columns: Record[] = KEYS.map(key => { - return { title: key, key }; - }); - - return { columns, rows: rows.filter((row, index) => index < 20) }; -}; From 363e65a280137d7fe7ac1922a00968f87ff0ca55 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Thu, 15 Sep 2022 16:53:34 +0200 Subject: [PATCH 04/17] Create tongaRawData custom report --- packages/report-server/package.json | 1 + .../customReports/data/tongaCovidRawData.json | 82 +++++ .../data/tongaCovidRawDataFaster.json | 38 +++ .../src/reportBuilder/customReports/index.ts | 8 +- .../customReports/tongaCovidRawData.ts | 305 ++++++++++++++++++ .../customReports/tongaCovidRawDataFaster.ts | 135 ++++++++ yarn.lock | 8 + 7 files changed, 576 insertions(+), 1 deletion(-) create mode 100644 packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json create mode 100644 packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json create mode 100644 packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts create mode 100644 packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts diff --git a/packages/report-server/package.json b/packages/report-server/package.json index 5ef33ea455..76c1ead6ea 100644 --- a/packages/report-server/package.json +++ b/packages/report-server/package.json @@ -39,6 +39,7 @@ "api-error-handler": "^1.0.0", "body-parser": "^1.18.3", "cors": "^2.8.5", + "date-fns": "^2.29.3", "dotenv": "^8.2.0", "express": "^4.16.2", "lodash.isplainobject": "^4.0.6", diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json new file mode 100644 index 0000000000..a456d570a4 --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json @@ -0,0 +1,82 @@ +{ + "C19T_Registration": { + "dataElementCodes": ["C19T002","C19T003","C19T004","C19T005","C19T006"] + }, + "C19T_Results": { + "dataElementCodes": ["C19T012","C19T013","C19T013_a","C19T015","C19T015_a","C19T016","C19T042","C19T017","C19T018","C19T019","C19T021","C19T022","C19T020","C19T038","C19T039","C19T044","C19T024","C19T025","C19T026","C19T027","C19T028","C19T041","C19T029","C19T033","C19T034","C19T035","C19T036","C19T037","C19T043"] + }, + "codesToNames": { + "C19T002": "Surname", + "C19T003": "Given Names", + "C19T004": "dob", + "C19T005": "Sex", + "C19T006": "Phone No.", + "C19T012": "Test Type", + "C19T013": "Other Type", + "C19T013_a":"CT Value", + "C19T015": "Outbound Traveller", + "C19T015_a": "Inbound Traveller", + "C19T016": "Symptomatic", + "C19T042": "Date of Symptom Onset", + "C19T017": "Quarantine", + "C19T018": "Primary Contact", + "C19T019": "Frontliner", + "C19T021": "Frontliner Type", + "C19T022": "Other", + "C19T020": "Community Testing", + "C19T038": "Patient", + "C19T039": "Other Reason", + "C19T044": "Primary Contact Testing Day", + "C19T024": "Testing Site", + "C19T025": "Quarantine Facility", + "C19T026": "Ward Type", + "C19T027": "Clinic Type", + "C19T028": "Health Center", + "C19T041": "RRT", + "C19T029": "Other Site", + "C19T033": "Result", + "C19T034": "Other Results", + "C19T035": "New Case", + "C19T036": "Previous Positive", + "C19T037": "Date Previous Positive", + "C19T043": "Vaccination Status" + }, + "columns": [ + "Test ID", + "Surname", + "Given Names", + "Date of Birth", + "Age", + "Sex", + "Phone No.", + "Island Group", + "Village Code", + "Address", + "Date of Test", + "Result", + "New Case", + "Estimated Recovery Date", + "Test Type", + "CT Value", + "Vaccination Status", + "Outbound Traveller", + "inbound Traveller", + "Symptomatic", + "Date of Symptomatic Onset", + "Quarantine", + "Primary Contact", + "Frontliner", + "Frontliner Type", + "Other", + "Community Testing", + "Patient", + "Other Reason", + "Primary Contact Testing Day", + "Testing Site", + "Quarantine Facility", + "Ward Type", + "Clinic Type", + "Health Center", + "RRT Team Name" + ] +} \ No newline at end of file diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json new file mode 100644 index 0000000000..09c7b49dd2 --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json @@ -0,0 +1,38 @@ +[ + "Test ID", + "Surname", + "Given Names", + "Date of Birth", + "Age", + "Sex", + "Phone No.", + "Island Group", + "Village Code", + "Address", + "Date of Test", + "Result", + "New Case", + "Estimated Recovery Date", + "Test Type", + "CT Value", + "Vaccination Status", + "Outbound Traveller", + "inbound Traveller", + "Symptomatic", + "Date of Symptomatic Onset", + "Quarantine", + "Primary Contact", + "Frontliner", + "Frontliner Type", + "Other", + "Community Testing", + "Patient", + "Other Reason", + "Primary Contact Testing Day", + "Testing Site", + "Quarantine Facility", + "Ward Type", + "Clinic Type", + "Health Center", + "RRT Team Name" +] \ No newline at end of file diff --git a/packages/report-server/src/reportBuilder/customReports/index.ts b/packages/report-server/src/reportBuilder/customReports/index.ts index cba577a298..07fead265c 100644 --- a/packages/report-server/src/reportBuilder/customReports/index.ts +++ b/packages/report-server/src/reportBuilder/customReports/index.ts @@ -7,10 +7,16 @@ import { Resolved } from '@tupaia/tsutils'; import { FetchReportQuery } from '../../types'; import { ReqContext } from '../context'; import { testCustomReport } from './testCustomReport'; +import { tongaCovidRawData } from './tongaCovidRawData'; +import { tongaCovidRawDataFaster } from './tongaCovidRawDataFaster'; type CustomReportBuilder = (reqContext: ReqContext, query: FetchReportQuery) => Promise; -export const customReports: Record = { testCustomReport }; +export const customReports: Record = { + testCustomReport, + tongaCovidRawData, + tongaCovidRawDataFaster, +}; export type CustomReportOutputType = Resolved< ReturnType diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts new file mode 100644 index 0000000000..5fd3aa380e --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -0,0 +1,305 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ +import { format, differenceInYears, addDays, isDate } from 'date-fns'; +import { createAggregator } from '@tupaia/aggregator'; +import { ReportServerAggregator } from '../../aggregator'; +import { FetchReportQuery, Event } from '../../types'; +import { ReqContext } from '../context'; +import SURVEYS from './data/tongaCovidRawData.json'; + +interface RelationshipsOptions { + hierarchy: string; + individualsByCode: string[]; + groupBy: 'ancestor' | 'descendant'; + queryOptions?: any; + ancestorOptions?: any; + descendantOptions?: any; +} + +interface Options { + programCode: string; + dataElementCodes: string[]; +} + +interface AncestorData { + villageByIndividual: Record>; + islandByIndividual: Record; +} + +const NOW = new Date(); + +const getRelationships = (reqContext: ReqContext, options: RelationshipsOptions) => { + const { + hierarchy, + individualsByCode, + groupBy, + queryOptions, + ancestorOptions, + descendantOptions, + } = options; + return reqContext.services.entity.getRelationshipsOfEntities( + hierarchy, + individualsByCode, + groupBy, + queryOptions, + ancestorOptions, + descendantOptions, + ); +}; + +const useAncestorData = async ( + reqContext: ReqContext, + hierarchy: string, + individualsByCode: string[], +) => { + const villageCodeByIndividualCodes: Record = await getRelationships(reqContext, { + hierarchy, + individualsByCode, + groupBy: 'descendant', + queryOptions: {}, + ancestorOptions: { filter: { type: 'village' } }, + descendantOptions: { filter: { type: 'individual' } }, + }); + const villageCodes = new Set(); + Object.values(villageCodeByIndividualCodes).forEach((code: string) => { + villageCodes.add(code); + }); + const includedVillageCodes = [...villageCodes]; + const villageCodesAndNames = await reqContext.services.entity.getEntities( + hierarchy, + includedVillageCodes, + { + fields: ['code', 'name'], + }, + ); + const individualCodeByVillageNameAndCode: Record> = {}; + Object.keys(villageCodeByIndividualCodes).forEach(individualCode => { + const villageCode = villageCodeByIndividualCodes[individualCode]; + const { name } = villageCodesAndNames.find( + (village: Record) => village.code === villageCode, + ); + individualCodeByVillageNameAndCode[individualCode] = { + code: villageCode, + name, + }; + }); + + const islandNameByIndividualCodes = await getRelationships(reqContext, { + hierarchy, + individualsByCode, + groupBy: 'descendant', + queryOptions: { field: 'name' }, + ancestorOptions: { filter: { type: 'district' } }, + descendantOptions: { filter: { type: 'individual' } }, + }); + + return { + villageByIndividual: individualCodeByVillageNameAndCode, + islandByIndividual: islandNameByIndividualCodes, + }; +}; + +const fetchEvents = async ( + reqContext: ReqContext, + individualsByCode: string[], + hierarchy: string, + startDate?: string, + endDate?: string, + period?: string, +) => { + const registrationOptions = { + programCode: 'C19T_Registration', + dataElementCodes: SURVEYS.C19T_Registration.dataElementCodes, + }; + const resultsOptions = { + programCode: 'C19T_Results', + dataElementCodes: SURVEYS.C19T_Results.dataElementCodes, + }; + + const aggregator = new ReportServerAggregator(createAggregator(undefined, reqContext)); + const fetch = async (options: Options) => { + const { programCode, dataElementCodes } = options; + return aggregator.fetchEvents( + programCode, + undefined, + individualsByCode, + hierarchy, + { startDate, endDate, period }, + dataElementCodes, + ); + }; + + const registrationEvents = await fetch(registrationOptions); + const resultsEvents = await fetch(resultsOptions); + return { registrationEvents, resultsEvents }; +}; + +const combineAndFlatten = ( + registrationEvents: Event[], + resultEvents: Event[], + ancestorData: AncestorData, +) => { + const matchedData: Record[] = resultEvents.map(resultEvent => { + const { dataValues: resultDataValues, orgUnit: resultOrgUnit, eventDate } = resultEvent; + const matchingRegistration = registrationEvents.find( + registrationEvent => registrationEvent.orgUnit === resultEvent.orgUnit, + ); + if (!matchingRegistration) { + return { + orgUnit: resultOrgUnit, + eventDate, + ...resultDataValues, + }; + } + const { dataValues: registrationDataValues, orgUnit } = matchingRegistration; + + return { + orgUnit, + eventDate, + ...registrationDataValues, + ...resultDataValues, + }; + }); + const { villageByIndividual, islandByIndividual } = ancestorData; + + const dataWithUpdatesAndAddOns = matchedData.map(event => { + const { + orgUnit, + C19T033: result, + C19T042: onsetDate, + C19T004: dob, + eventDate: dateSpecimenCollected, + } = event; + const { code: villageCode, name: villageName } = villageByIndividual[ + orgUnit as keyof typeof Event + ]; + const islandName: string = islandByIndividual[orgUnit]; + const age = getAge(dob); + return { + 'Test ID': orgUnit, + 'Island Group': islandName, + 'Village Code': villageCode, + Address: villageName, + 'Estimated Recovery Date': getEstimatedRecoveryDate(result, dateSpecimenCollected, onsetDate), + dateSpecimenCollected, + Age: age, + ...event, + }; + }); + + return dataWithUpdatesAndAddOns; +}; + +const getAge = (dob: string | undefined) => { + if (!dob) { + return 'unknown'; + } + const dobDate = new Date(dob); + const age = isDate(dobDate) && differenceInYears(NOW, dobDate); + return age; +}; + +const getEstimatedRecoveryDate = ( + result: string, + collectionDate: string, + onsetDate: string | undefined, +) => { + if (result !== 'Positive') { + return 'Not applicable'; + } + + if (onsetDate) { + const recoveryDate = addDays(new Date(onsetDate), 13); + + return isDate(recoveryDate) && format(recoveryDate, 'yyy-mm-dd'); + } + + const recoveryDate = addDays(new Date(collectionDate), 13); + return isDate(recoveryDate) && format(recoveryDate, 'yyyy-mm-dd'); +}; + +const parseRowData = (rowData: Record) => { + const formattedRow: Record = {}; + const { codesToNames } = SURVEYS; + Object.keys(rowData).forEach(fieldKey => { + switch (fieldKey) { + case 'dateSpecimenCollected': { + formattedRow['Date of Test'] = rowData[fieldKey]; + break; + } + case 'C19T004': { + if (!rowData[fieldKey]) { + formattedRow['Date of Birth'] = 'Unknown'; + } + const rawDate = new Date(rowData[fieldKey]); + const dobValue = isDate(rawDate) && format(rawDate, 'yyyy-mm-dd'); + formattedRow['Date of Birth'] = dobValue; + break; + } + case 'Test ID': + case 'Island Group': + case 'Village Code': + case 'Address': + case 'Age': + case 'Estimated Recovery Date': + formattedRow[fieldKey] = rowData[fieldKey]; + break; + default: { + const name = codesToNames[fieldKey as keyof typeof codesToNames]; + if (!name) { + formattedRow[fieldKey] = rowData[fieldKey]; + } else { + formattedRow[name] = rowData[fieldKey]; + } + } + } + }); + return formattedRow; +}; + +export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchReportQuery) => { + const { organisationUnitCodes: entityCodes, hierarchy, startDate, endDate, period } = query; + const individuals = await reqContext.services.entity.getDescendantsOfEntities( + hierarchy, + entityCodes, + { filter: { type: 'individual' } }, + ); + const individualsByCode = individuals.map((ind: Record) => ind.code); + + const ancestorData: AncestorData = await useAncestorData( + reqContext, + hierarchy, + individualsByCode, + ); + + const { registrationEvents, resultsEvents } = await fetchEvents( + reqContext, + individualsByCode, + hierarchy, + startDate, + endDate, + period, + ); + + const builtEvents: Record[] = combineAndFlatten( + registrationEvents, + resultsEvents, + ancestorData, + ); + const rows = builtEvents + .map(rowData => parseRowData(rowData)) + .sort((row, nextRow) => { + if (row['Test ID'] === nextRow['Test ID']) { + return 0; + } + return 1; + }); + + const columns: Record[] = SURVEYS.columns.map(key => { + return { title: key, key }; + }); + + return { columns, rows }; +}; diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts new file mode 100644 index 0000000000..547a937edb --- /dev/null +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts @@ -0,0 +1,135 @@ +import { TupaiaDatabase } from '@tupaia/database'; +import { string } from 'mathjs'; +import KEYS from './data/tongaCovidRawDataFaster.json'; + +export const tongaCovidRawDataFaster = async () => { + const db = new TupaiaDatabase(); + const rows: Record[] = await db.executeSql( + ` + select + C19.lab_id as "Test ID", + C19.surname as "Surname", + C19.first_name as "Given Names", + C19.dob as "Date of Birth", + C19.age as "Age", + C19.sex as "Sex", + C19.phone as "Phone No.", + C19.island as "Island Group", + C19.village_code as "Village Code", + C19.village as "Address", + C19.date_specimen_collected as "Date of Test", + C19.results as "Result", + C19.new_case as "New Case", + (case + when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is not null then (C19.date_symptom_onset::date + interval '13' day)::date + when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is null then (C19.date_specimen_collected + interval '13' day)::date + when C19.results = 'Positive' then (C19.date_specimen_collected + interval '10' day)::date + end) as "Estimated Recovery Date", + C19.test_type as "Test Type", + C19.ctvalue as "CT Value", + C19.vax_status as "Vaccination Status", + C19.outbound_traveller as "Outbound Traveller", + C19.inbound_traveller as "Inbound Traveller", + C19.symptomatic as "Symptomatic", + C19.date_symptom_onset as "Date of Symptomatic Onset", + C19.quarantine as "Quarantine", + C19.linkedtocase as "Primary Contact", + C19.frontliner as "Frontliner", + C19.frontliner_type as "Frontliner Type", + C19.other_frontliner_type as "Other", + C19.communitytesting as "Community Testing", + C19.patient as "Patient", + C19.other_reason as "Other Reason", + C19.primarycontact_testingday as "Primary Contact Testing Day", + C19.specimen_site as "Testing Site", + C19.quarantine_facility as "Quarantine Facility", + C19.ward_type as "Ward Type", + C19.clinic_type as "Clinic Type", + hc.name as "Health Center", + C19.rrt_team as "RRT Team Name", + C19.other_site as "Other" + from + ( + select + e.code as lab_id, + reg.date as date_registered, + reg.surname, + reg.first_name, + reg.dob, + date_part('year', age(sr.data_time::date, reg.dob::date)) as age, + reg.sex, + reg.phone, + ggp.name as island, + p.code as village_code, + p.name as village, + sr.data_time as date_specimen_collected, + max(case when q.code = 'C19T012' then a.text end) as test_type, + max(case when q.code = 'C19T013' then a.text end) as other_type, + max(case when q.code = 'C19T013_a' then a.text end) as ctvalue, + max(case when q.code = 'C19T015' then a.text end) as outbound_traveller, + max(case when q.code = 'C19T015_a' then a.text end) as inbound_traveller, + max(case when q.code = 'C19T016' then a.text end) as symptomatic, + max(case when q.code = 'C19T042' then a.text end) as date_symptom_onset, + max(case when q.code = 'C19T017' then a.text end) as quarantine, + max(case when q.code = 'C19T018' then a.text end) as linkedtocase, + max(case when q.code = 'C19T019' then a.text end) as frontliner, + max(case when q.code = 'C19T021' then a.text end) as frontliner_type, + max(case when q.code = 'C19T022' then a.text end) as other_frontliner_type, + max(case when q.code = 'C19T020' then a.text end) as communitytesting, + max(case when q.code = 'C19T038' then a.text end) as patient, + max(case when q.code = 'C19T039' then a.text end) as other_reason, + max(case when q.code = 'C19T044' then a.text end) as primarycontact_testingday, + max(case when q.code = 'C19T024' then a.text end) as specimen_site, + max(case when q.code = 'C19T025' then a.text end) as quarantine_facility, + max(case when q.code = 'C19T026' then a.text end) as ward_type, + max(case when q.code = 'C19T027' then a.text end) as clinic_type, + max(case when q.code = 'C19T028' then a.text end) as health_centre, + max(case when q.code = 'C19T041' then a.text end) as rrt_team, + max(case when q.code = 'C19T029' then a.text end) as other_site, + max(case when q.code = 'C19T032' then a.text end) as rat, + max(case when q.code = 'C19T033' then a.text end) as results, + max(case when q.code = 'C19T034' then a.text end) as other_results, + max(case when q.code = 'C19T035' then a.text end) as new_case, + max(case when q.code = 'C19T036' then a.text end) as previous_positive, + max(case when q.code = 'C19T037' then to_char(a.text::timestamp::date, 'yyyy-mm-dd') end) as date_previous_positive, + max(case when q.code = 'C19T043' then a.text end) as vax_status + from survey_response sr + join answer a on a.survey_response_id = sr.id + join question q on q.id = a.question_id + join survey s on s.id = sr.survey_id + join entity e on e.id = sr.entity_id + join entity p on p.id = e.parent_id + join entity gp on gp.id = p.parent_id + join entity ggp on ggp.id = gp.parent_id + join + ( + select + e1.code as entity_code, + sr1.data_time as date, + max(case when q1.code = 'C19T002' then a1.text end) as surname, + max(case when q1.code = 'C19T003' then a1.text end) as first_name, + max(case when q1.code = 'C19T004' then to_char(a1.text::timestamp::date, 'yyyy-mm-dd') end) as dob, + max(case when q1.code = 'C19T005' then a1.text end) as sex, + max(case when q1.code = 'C19T006' then a1.text end) as phone + from survey_response sr1 + join answer a1 on a1.survey_response_id = sr1.id + join question q1 on q1.id = a1.question_id + join survey s1 on s1.id = sr1.survey_id + join entity e1 on e1.id = sr1.entity_id + where s1.code = 'C19T_Registration' and e1.country_code = 'TO' + group by e1.code, sr1.data_time + ) reg on reg.entity_code = e.code + where s.code = 'C19T_Results' and e.country_code = 'TO' + group by sr.id, reg.date, reg.surname, reg.first_name, reg.dob, reg.sex, reg.phone, sr.data_time, e.code, p.name, p.code, ggp.name + ) C19 + left join entity hc on hc.id = C19.health_centre + `, + [], + ); + + const columns: Record[] = KEYS.map(key => { + return { title: key, key }; + }); + + return { columns, rows: rows.filter((row, index) => index < 20) }; +}; diff --git a/yarn.lock b/yarn.lock index 8af8885d85..8c65171945 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5580,6 +5580,7 @@ __metadata: api-error-handler: ^1.0.0 body-parser: ^1.18.3 cors: ^2.8.5 + date-fns: ^2.29.3 dotenv: ^8.2.0 express: ^4.16.2 lodash.isplainobject: ^4.0.6 @@ -12641,6 +12642,13 @@ __metadata: languageName: node linkType: hard +"date-fns@npm:^2.29.3": + version: 2.29.3 + resolution: "date-fns@npm:2.29.3" + checksum: e01cf5b62af04e05dfff921bb9c9933310ed0e1ae9a81eb8653452e64dc841acf7f6e01e1a5ae5644d0337e9a7f936175fd2cb6819dc122fdd9c5e86c56be484 + languageName: node + linkType: hard + "dayjs@npm:^1.8.15": version: 1.9.6 resolution: "dayjs@npm:1.9.6" From 418515fa265301fc4f3a3e155726453a6248b6a4 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Fri, 16 Sep 2022 08:41:55 +0200 Subject: [PATCH 05/17] Run fetches through promise --- .../customReports/tongaCovidRawData.ts | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts index 5fd3aa380e..9fb2321753 100644 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -28,8 +28,6 @@ interface AncestorData { islandByIndividual: Record; } -const NOW = new Date(); - const getRelationships = (reqContext: ReqContext, options: RelationshipsOptions) => { const { hierarchy, @@ -54,14 +52,27 @@ const useAncestorData = async ( hierarchy: string, individualsByCode: string[], ) => { - const villageCodeByIndividualCodes: Record = await getRelationships(reqContext, { + const villageOptions: RelationshipsOptions = { hierarchy, individualsByCode, groupBy: 'descendant', queryOptions: {}, ancestorOptions: { filter: { type: 'village' } }, descendantOptions: { filter: { type: 'individual' } }, - }); + }; + + const islandOptions: RelationshipsOptions = { + hierarchy, + individualsByCode, + groupBy: 'descendant', + queryOptions: { field: 'name' }, + ancestorOptions: { filter: { type: 'district' } }, + descendantOptions: { filter: { type: 'individual' } }, + }; + const villageCodeByIndividualCodes: Record = await getRelationships( + reqContext, + villageOptions, + ); const villageCodes = new Set(); Object.values(villageCodeByIndividualCodes).forEach((code: string) => { villageCodes.add(code); @@ -86,14 +97,7 @@ const useAncestorData = async ( }; }); - const islandNameByIndividualCodes = await getRelationships(reqContext, { - hierarchy, - individualsByCode, - groupBy: 'descendant', - queryOptions: { field: 'name' }, - ancestorOptions: { filter: { type: 'district' } }, - descendantOptions: { filter: { type: 'individual' } }, - }); + const islandNameByIndividualCodes = await getRelationships(reqContext, islandOptions); return { villageByIndividual: individualCodeByVillageNameAndCode, @@ -131,8 +135,10 @@ const fetchEvents = async ( ); }; - const registrationEvents = await fetch(registrationOptions); - const resultsEvents = await fetch(resultsOptions); + const [registrationEvents, resultsEvents] = await Promise.all( + [registrationOptions, resultsOptions].map(fetch), + ); + return { registrationEvents, resultsEvents }; }; @@ -163,7 +169,7 @@ const combineAndFlatten = ( }; }); const { villageByIndividual, islandByIndividual } = ancestorData; - + const now = new Date(); const dataWithUpdatesAndAddOns = matchedData.map(event => { const { orgUnit, @@ -176,7 +182,8 @@ const combineAndFlatten = ( orgUnit as keyof typeof Event ]; const islandName: string = islandByIndividual[orgUnit]; - const age = getAge(dob); + + const age = getAge(dob, now); return { 'Test ID': orgUnit, 'Island Group': islandName, @@ -192,12 +199,12 @@ const combineAndFlatten = ( return dataWithUpdatesAndAddOns; }; -const getAge = (dob: string | undefined) => { +const getAge = (dob: string | undefined, now: Date) => { if (!dob) { return 'unknown'; } const dobDate = new Date(dob); - const age = isDate(dobDate) && differenceInYears(NOW, dobDate); + const age = isDate(dobDate) && differenceInYears(now, dobDate); return age; }; @@ -301,5 +308,5 @@ export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchRepo return { title: key, key }; }); - return { columns, rows }; + return { columns, rows: rows.filter((row, index) => index < 20) }; }; From 362f27903a76a148a226793b5c2dbecdc0c6f5bf Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Fri, 16 Sep 2022 11:51:36 +0200 Subject: [PATCH 06/17] Delete SQL version --- .../data/tongaCovidRawDataFaster.json | 38 ----- .../customReports/tongaCovidRawDataFaster.ts | 135 ------------------ 2 files changed, 173 deletions(-) delete mode 100644 packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json delete mode 100644 packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json deleted file mode 100644 index 09c7b49dd2..0000000000 --- a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawDataFaster.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - "Test ID", - "Surname", - "Given Names", - "Date of Birth", - "Age", - "Sex", - "Phone No.", - "Island Group", - "Village Code", - "Address", - "Date of Test", - "Result", - "New Case", - "Estimated Recovery Date", - "Test Type", - "CT Value", - "Vaccination Status", - "Outbound Traveller", - "inbound Traveller", - "Symptomatic", - "Date of Symptomatic Onset", - "Quarantine", - "Primary Contact", - "Frontliner", - "Frontliner Type", - "Other", - "Community Testing", - "Patient", - "Other Reason", - "Primary Contact Testing Day", - "Testing Site", - "Quarantine Facility", - "Ward Type", - "Clinic Type", - "Health Center", - "RRT Team Name" -] \ No newline at end of file diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts deleted file mode 100644 index 547a937edb..0000000000 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawDataFaster.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { TupaiaDatabase } from '@tupaia/database'; -import { string } from 'mathjs'; -import KEYS from './data/tongaCovidRawDataFaster.json'; - -export const tongaCovidRawDataFaster = async () => { - const db = new TupaiaDatabase(); - const rows: Record[] = await db.executeSql( - ` - select - C19.lab_id as "Test ID", - C19.surname as "Surname", - C19.first_name as "Given Names", - C19.dob as "Date of Birth", - C19.age as "Age", - C19.sex as "Sex", - C19.phone as "Phone No.", - C19.island as "Island Group", - C19.village_code as "Village Code", - C19.village as "Address", - C19.date_specimen_collected as "Date of Test", - C19.results as "Result", - C19.new_case as "New Case", - (case - when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is not null then (C19.date_symptom_onset::date + interval '13' day)::date - when C19.results = 'Positive' and C19.symptomatic = 'Yes' and C19.date_symptom_onset is null then (C19.date_specimen_collected + interval '13' day)::date - when C19.results = 'Positive' then (C19.date_specimen_collected + interval '10' day)::date - end) as "Estimated Recovery Date", - C19.test_type as "Test Type", - C19.ctvalue as "CT Value", - C19.vax_status as "Vaccination Status", - C19.outbound_traveller as "Outbound Traveller", - C19.inbound_traveller as "Inbound Traveller", - C19.symptomatic as "Symptomatic", - C19.date_symptom_onset as "Date of Symptomatic Onset", - C19.quarantine as "Quarantine", - C19.linkedtocase as "Primary Contact", - C19.frontliner as "Frontliner", - C19.frontliner_type as "Frontliner Type", - C19.other_frontliner_type as "Other", - C19.communitytesting as "Community Testing", - C19.patient as "Patient", - C19.other_reason as "Other Reason", - C19.primarycontact_testingday as "Primary Contact Testing Day", - C19.specimen_site as "Testing Site", - C19.quarantine_facility as "Quarantine Facility", - C19.ward_type as "Ward Type", - C19.clinic_type as "Clinic Type", - hc.name as "Health Center", - C19.rrt_team as "RRT Team Name", - C19.other_site as "Other" - from - ( - select - e.code as lab_id, - reg.date as date_registered, - reg.surname, - reg.first_name, - reg.dob, - date_part('year', age(sr.data_time::date, reg.dob::date)) as age, - reg.sex, - reg.phone, - ggp.name as island, - p.code as village_code, - p.name as village, - sr.data_time as date_specimen_collected, - max(case when q.code = 'C19T012' then a.text end) as test_type, - max(case when q.code = 'C19T013' then a.text end) as other_type, - max(case when q.code = 'C19T013_a' then a.text end) as ctvalue, - max(case when q.code = 'C19T015' then a.text end) as outbound_traveller, - max(case when q.code = 'C19T015_a' then a.text end) as inbound_traveller, - max(case when q.code = 'C19T016' then a.text end) as symptomatic, - max(case when q.code = 'C19T042' then a.text end) as date_symptom_onset, - max(case when q.code = 'C19T017' then a.text end) as quarantine, - max(case when q.code = 'C19T018' then a.text end) as linkedtocase, - max(case when q.code = 'C19T019' then a.text end) as frontliner, - max(case when q.code = 'C19T021' then a.text end) as frontliner_type, - max(case when q.code = 'C19T022' then a.text end) as other_frontliner_type, - max(case when q.code = 'C19T020' then a.text end) as communitytesting, - max(case when q.code = 'C19T038' then a.text end) as patient, - max(case when q.code = 'C19T039' then a.text end) as other_reason, - max(case when q.code = 'C19T044' then a.text end) as primarycontact_testingday, - max(case when q.code = 'C19T024' then a.text end) as specimen_site, - max(case when q.code = 'C19T025' then a.text end) as quarantine_facility, - max(case when q.code = 'C19T026' then a.text end) as ward_type, - max(case when q.code = 'C19T027' then a.text end) as clinic_type, - max(case when q.code = 'C19T028' then a.text end) as health_centre, - max(case when q.code = 'C19T041' then a.text end) as rrt_team, - max(case when q.code = 'C19T029' then a.text end) as other_site, - max(case when q.code = 'C19T032' then a.text end) as rat, - max(case when q.code = 'C19T033' then a.text end) as results, - max(case when q.code = 'C19T034' then a.text end) as other_results, - max(case when q.code = 'C19T035' then a.text end) as new_case, - max(case when q.code = 'C19T036' then a.text end) as previous_positive, - max(case when q.code = 'C19T037' then to_char(a.text::timestamp::date, 'yyyy-mm-dd') end) as date_previous_positive, - max(case when q.code = 'C19T043' then a.text end) as vax_status - from survey_response sr - join answer a on a.survey_response_id = sr.id - join question q on q.id = a.question_id - join survey s on s.id = sr.survey_id - join entity e on e.id = sr.entity_id - join entity p on p.id = e.parent_id - join entity gp on gp.id = p.parent_id - join entity ggp on ggp.id = gp.parent_id - join - ( - select - e1.code as entity_code, - sr1.data_time as date, - max(case when q1.code = 'C19T002' then a1.text end) as surname, - max(case when q1.code = 'C19T003' then a1.text end) as first_name, - max(case when q1.code = 'C19T004' then to_char(a1.text::timestamp::date, 'yyyy-mm-dd') end) as dob, - max(case when q1.code = 'C19T005' then a1.text end) as sex, - max(case when q1.code = 'C19T006' then a1.text end) as phone - from survey_response sr1 - join answer a1 on a1.survey_response_id = sr1.id - join question q1 on q1.id = a1.question_id - join survey s1 on s1.id = sr1.survey_id - join entity e1 on e1.id = sr1.entity_id - where s1.code = 'C19T_Registration' and e1.country_code = 'TO' - group by e1.code, sr1.data_time - ) reg on reg.entity_code = e.code - where s.code = 'C19T_Results' and e.country_code = 'TO' - group by sr.id, reg.date, reg.surname, reg.first_name, reg.dob, reg.sex, reg.phone, sr.data_time, e.code, p.name, p.code, ggp.name - ) C19 - left join entity hc on hc.id = C19.health_centre - `, - [], - ); - - const columns: Record[] = KEYS.map(key => { - return { title: key, key }; - }); - - return { columns, rows: rows.filter((row, index) => index < 20) }; -}; From 2f97ba8908ee1103ab1d0affcbeb94c5afe694dd Mon Sep 17 00:00:00 2001 From: Rohan Port Date: Mon, 19 Sep 2022 10:15:50 +1000 Subject: [PATCH 07/17] MAUI-775: Removed references to SQL export method --- packages/report-server/src/reportBuilder/customReports/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/report-server/src/reportBuilder/customReports/index.ts b/packages/report-server/src/reportBuilder/customReports/index.ts index 07fead265c..87eab70f87 100644 --- a/packages/report-server/src/reportBuilder/customReports/index.ts +++ b/packages/report-server/src/reportBuilder/customReports/index.ts @@ -8,14 +8,12 @@ import { FetchReportQuery } from '../../types'; import { ReqContext } from '../context'; import { testCustomReport } from './testCustomReport'; import { tongaCovidRawData } from './tongaCovidRawData'; -import { tongaCovidRawDataFaster } from './tongaCovidRawDataFaster'; type CustomReportBuilder = (reqContext: ReqContext, query: FetchReportQuery) => Promise; export const customReports: Record = { testCustomReport, tongaCovidRawData, - tongaCovidRawDataFaster, }; export type CustomReportOutputType = Resolved< From bb7fd8b672faee8649f7042a9c6952d14aec0637 Mon Sep 17 00:00:00 2001 From: Rohan Port Date: Wed, 21 Sep 2022 16:28:48 +1000 Subject: [PATCH 08/17] MAUI-775: Code cleanups and performance improvements to tongaCovidRawData custom report --- .../customReports/tongaCovidRawData.ts | 139 +++++++++--------- 1 file changed, 67 insertions(+), 72 deletions(-) diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts index 9fb2321753..68b8e99efe 100644 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -11,8 +11,7 @@ import SURVEYS from './data/tongaCovidRawData.json'; interface RelationshipsOptions { hierarchy: string; - individualsByCode: string[]; - groupBy: 'ancestor' | 'descendant'; + entityCodes: string[]; queryOptions?: any; ancestorOptions?: any; descendantOptions?: any; @@ -23,39 +22,26 @@ interface Options { dataElementCodes: string[]; } -interface AncestorData { - villageByIndividual: Record>; - islandByIndividual: Record; -} - const getRelationships = (reqContext: ReqContext, options: RelationshipsOptions) => { - const { - hierarchy, - individualsByCode, - groupBy, - queryOptions, - ancestorOptions, - descendantOptions, - } = options; + const { hierarchy, entityCodes, queryOptions, ancestorOptions, descendantOptions } = options; return reqContext.services.entity.getRelationshipsOfEntities( hierarchy, - individualsByCode, - groupBy, + entityCodes, + 'descendant', queryOptions, ancestorOptions, descendantOptions, - ); + ) as Promise>; }; -const useAncestorData = async ( +const fetchVillagesAndIslands = async ( reqContext: ReqContext, hierarchy: string, - individualsByCode: string[], + entityCodes: string[], ) => { const villageOptions: RelationshipsOptions = { hierarchy, - individualsByCode, - groupBy: 'descendant', + entityCodes, queryOptions: {}, ancestorOptions: { filter: { type: 'village' } }, descendantOptions: { filter: { type: 'individual' } }, @@ -63,42 +49,43 @@ const useAncestorData = async ( const islandOptions: RelationshipsOptions = { hierarchy, - individualsByCode, - groupBy: 'descendant', - queryOptions: { field: 'name' }, - ancestorOptions: { filter: { type: 'district' } }, + entityCodes, + queryOptions: {}, + ancestorOptions: { field: 'name', filter: { type: 'district' } }, descendantOptions: { filter: { type: 'individual' } }, }; - const villageCodeByIndividualCodes: Record = await getRelationships( - reqContext, - villageOptions, + const [villageCodeByIndividualCodes, islandNameByIndividualCodes] = await Promise.all( + [villageOptions, islandOptions].map(options => getRelationships(reqContext, options)), ); + const villageCodes = new Set(); Object.values(villageCodeByIndividualCodes).forEach((code: string) => { villageCodes.add(code); }); const includedVillageCodes = [...villageCodes]; - const villageCodesAndNames = await reqContext.services.entity.getEntities( + const villageCodesAndNames = (await reqContext.services.entity.getEntities( hierarchy, includedVillageCodes, { fields: ['code', 'name'], }, - ); - const individualCodeByVillageNameAndCode: Record> = {}; + )) as { code: string; name: string }[]; + const individualCodeByVillageNameAndCode: Record = {}; Object.keys(villageCodeByIndividualCodes).forEach(individualCode => { const villageCode = villageCodeByIndividualCodes[individualCode]; - const { name } = villageCodesAndNames.find( - (village: Record) => village.code === villageCode, - ); + const villageWithCode = villageCodesAndNames.find(village => village.code === villageCode); + + if (!villageWithCode) { + throw new Error(`Could not find village with code: ${villageCode}`); + } + + const { name } = villageWithCode; individualCodeByVillageNameAndCode[individualCode] = { code: villageCode, name, }; }); - const islandNameByIndividualCodes = await getRelationships(reqContext, islandOptions); - return { villageByIndividual: individualCodeByVillageNameAndCode, islandByIndividual: islandNameByIndividualCodes, @@ -107,7 +94,7 @@ const useAncestorData = async ( const fetchEvents = async ( reqContext: ReqContext, - individualsByCode: string[], + entityCodes: string[], hierarchy: string, startDate?: string, endDate?: string, @@ -128,7 +115,7 @@ const fetchEvents = async ( return aggregator.fetchEvents( programCode, undefined, - individualsByCode, + entityCodes, hierarchy, { startDate, endDate, period }, dataElementCodes, @@ -142,11 +129,7 @@ const fetchEvents = async ( return { registrationEvents, resultsEvents }; }; -const combineAndFlatten = ( - registrationEvents: Event[], - resultEvents: Event[], - ancestorData: AncestorData, -) => { +const combineAndFlatten = (registrationEvents: Event[], resultEvents: Event[]) => { const matchedData: Record[] = resultEvents.map(resultEvent => { const { dataValues: resultDataValues, orgUnit: resultOrgUnit, eventDate } = resultEvent; const matchingRegistration = registrationEvents.find( @@ -168,7 +151,6 @@ const combineAndFlatten = ( ...resultDataValues, }; }); - const { villageByIndividual, islandByIndividual } = ancestorData; const now = new Date(); const dataWithUpdatesAndAddOns = matchedData.map(event => { const { @@ -178,17 +160,10 @@ const combineAndFlatten = ( C19T004: dob, eventDate: dateSpecimenCollected, } = event; - const { code: villageCode, name: villageName } = villageByIndividual[ - orgUnit as keyof typeof Event - ]; - const islandName: string = islandByIndividual[orgUnit]; const age = getAge(dob, now); return { 'Test ID': orgUnit, - 'Island Group': islandName, - 'Village Code': villageCode, - Address: villageName, 'Estimated Recovery Date': getEstimatedRecoveryDate(result, dateSpecimenCollected, onsetDate), dateSpecimenCollected, Age: age, @@ -246,9 +221,6 @@ const parseRowData = (rowData: Record) => { break; } case 'Test ID': - case 'Island Group': - case 'Village Code': - case 'Address': case 'Age': case 'Estimated Recovery Date': formattedRow[fieldKey] = rowData[fieldKey]; @@ -266,35 +238,43 @@ const parseRowData = (rowData: Record) => { return formattedRow; }; +const addVillageAndIsland = ( + rows: Record[], + villageByIndividual: Record, + islandByIndividual: Record, +) => { + return rows.map(row => { + const { orgUnit } = row; + const { code: villageCode, name: villageName } = villageByIndividual[orgUnit]; + const islandName = islandByIndividual[orgUnit]; + return { + ...row, + 'Village Code': villageCode, + Address: villageName, + 'Island Group': islandName, + }; + }); +}; + export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchReportQuery) => { const { organisationUnitCodes: entityCodes, hierarchy, startDate, endDate, period } = query; - const individuals = await reqContext.services.entity.getDescendantsOfEntities( - hierarchy, - entityCodes, - { filter: { type: 'individual' } }, - ); - const individualsByCode = individuals.map((ind: Record) => ind.code); - const ancestorData: AncestorData = await useAncestorData( - reqContext, + const individualCodes = await reqContext.services.entity.getDescendantsOfEntities( hierarchy, - individualsByCode, + entityCodes, + { field: 'code', filter: { type: 'individual' } }, ); const { registrationEvents, resultsEvents } = await fetchEvents( reqContext, - individualsByCode, + individualCodes, hierarchy, startDate, endDate, period, ); - const builtEvents: Record[] = combineAndFlatten( - registrationEvents, - resultsEvents, - ancestorData, - ); + const builtEvents: Record[] = combineAndFlatten(registrationEvents, resultsEvents); const rows = builtEvents .map(rowData => parseRowData(rowData)) .sort((row, nextRow) => { @@ -302,11 +282,26 @@ export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchRepo return 0; } return 1; - }); + }) + .filter((row, index) => index < 20); + + const individualsInRows = Array.from(new Set(rows.map(row => row.orgUnit))); + + const { villageByIndividual, islandByIndividual } = await fetchVillagesAndIslands( + reqContext, + hierarchy, + individualsInRows, + ); + + const rowsWithVillageAndIsland = addVillageAndIsland( + rows, + villageByIndividual, + islandByIndividual, + ); const columns: Record[] = SURVEYS.columns.map(key => { return { title: key, key }; }); - return { columns, rows: rows.filter((row, index) => index < 20) }; + return { columns, rows: rowsWithVillageAndIsland }; }; From 016b2378beae5944238b88ee5671f7a2618fe33a Mon Sep 17 00:00:00 2001 From: Rohan Port Date: Sat, 24 Sep 2022 16:21:03 +1000 Subject: [PATCH 09/17] MAUI-775: Updated entityApiMock --- .../reportBuilder/testUtils/entityApiMock.ts | 148 ++++++++++++++---- 1 file changed, 121 insertions(+), 27 deletions(-) diff --git a/packages/report-server/src/__tests__/reportBuilder/testUtils/entityApiMock.ts b/packages/report-server/src/__tests__/reportBuilder/testUtils/entityApiMock.ts index ad7a7e5b6e..fc466e6620 100644 --- a/packages/report-server/src/__tests__/reportBuilder/testUtils/entityApiMock.ts +++ b/packages/report-server/src/__tests__/reportBuilder/testUtils/entityApiMock.ts @@ -9,16 +9,43 @@ export const entityApiMock = ( entities: Record[]>, relations: Record = {}, ) => { + const getEntities = ( + hierarchyName: string, + entityCodes: string[], + queryOptions: { field?: string; fields?: string[]; filter?: Record } = {}, + ) => { + const entitiesInHierarchy = entities[hierarchyName] || []; + const foundEntities = entitiesInHierarchy.filter(e => entityCodes.includes(e.code)); + const { field, fields, filter } = queryOptions; + + let filteredEntities = foundEntities; + if (filter) { + filteredEntities = filteredEntities.filter(e => + Object.entries(filter).every(([key, value]) => e[key] === value), + ); + } + + if (field) { + return filteredEntities.map(e => e[field]); + } + + if (fields) { + return filteredEntities.map(e => pick(e, fields)); + } + + return filteredEntities; + }; + const getDescendantsOfEntities = ( hierarchyName: string, entityCodes: string[], - queryOptions: { fields?: string[]; filter?: { type: string } } = {}, + queryOptions: { fields?: string[]; filter?: { type?: string } } = {}, ) => { const entitiesInHierarchy = entities[hierarchyName] || []; const relationsInHierarchy = relations[hierarchyName] || []; const ancestorEntities = entitiesInHierarchy.filter(e => entityCodes.includes(e.code)); const ancestorEntityQueue = [...ancestorEntities]; - const descendantEntities = []; + const descendantEntityCodes = []; while (ancestorEntityQueue.length > 0) { const parent = ancestorEntityQueue.shift(); if (parent === undefined) { @@ -31,17 +58,52 @@ export const entityApiMock = ( .map(({ child }) => entitiesInHierarchy.find(entity => entity.code === child)) .filter(isDefined); - descendantEntities.push(...children); + descendantEntityCodes.push(...children.map(e => e.code)); ancestorEntityQueue.push(...children); } - const { fields, filter } = queryOptions; + return getEntities(hierarchyName, descendantEntityCodes, queryOptions); + }; + + const getAncestorsOfEntities = ( + hierarchyName: string, + entityCodes: string[], + queryOptions: { fields?: string[]; filter?: { type?: string } } = {}, + ) => { + const entitiesInHierarchy = entities[hierarchyName] || []; + const relationsInHierarchy = relations[hierarchyName] || []; + const descendantEntities = entitiesInHierarchy.filter(e => entityCodes.includes(e.code)); + const descendantEntityQueue = [...descendantEntities]; + const ancestorCodes = []; + while (descendantEntityQueue.length > 0) { + const child = descendantEntityQueue.shift(); + if (child === undefined) { + continue; + } + + const isDefined = (val: T): val is Exclude => val !== undefined; + const parents = relationsInHierarchy + .filter(({ child: childCode }) => child.code === childCode) + .map(({ parent }) => entitiesInHierarchy.find(entity => entity.code === parent)) + .filter(isDefined); + + ancestorCodes.push(...parents.map(e => e.code)); + descendantEntityQueue.push(...parents); + } - const filteredDescendants = filter - ? descendantEntities.filter(entity => entity.type === filter.type) - : descendantEntities; + return getEntities(hierarchyName, ancestorCodes, queryOptions); + }; - return fields ? filteredDescendants.map(e => pick(e, fields)) : filteredDescendants; + const getRelationsOfEntities = ( + hierarchyName: string, + entityCodes: string[], + queryOptions: { fields?: string[]; filter?: { type?: string } } = {}, + ) => { + return [ + ...getAncestorsOfEntities(hierarchyName, entityCodes, queryOptions), + ...getEntities(hierarchyName, entityCodes, queryOptions), + ...getDescendantsOfEntities(hierarchyName, entityCodes, queryOptions), + ]; }; return { @@ -49,11 +111,7 @@ export const entityApiMock = ( hierarchyName: string, entityCodes: string[], queryOptions: { fields?: string[] } = {}, - ) => { - const foundEntities = entities[hierarchyName]?.filter(e => entityCodes.includes(e.code)); - const { fields } = queryOptions; - return fields ? foundEntities.map(e => pick(e, fields)) : foundEntities; - }, + ) => getEntities(hierarchyName, entityCodes, queryOptions), getDescendantsOfEntities: async ( hierarchyName: string, entityCodes: string[], @@ -62,21 +120,57 @@ export const entityApiMock = ( getRelationshipsOfEntities: async ( hierarchyName: string, entityCodes: string[], - groupBy: 'ancestor', - queryOptions: { fields?: string[]; filter?: { type: string } } = {}, - ancestorQueryOptions: { filter?: { type: string } } = {}, - descendantQueryOptions: { filter?: { type: string } } = {}, + groupBy: 'ancestor' | 'descendant', + queryOptions: { field?: string } = {}, + ancestorQueryOptions: { field?: string; filter?: { type: string } } = {}, + descendantQueryOptions: { field?: string; filter?: { type: string } } = {}, ) => { - const entitiesInHierarchy = entities[hierarchyName] || []; - const ancestorEntities = ancestorQueryOptions.filter?.type - ? getDescendantsOfEntities(hierarchyName, entityCodes, ancestorQueryOptions) - : entitiesInHierarchy.filter(e => entityCodes.includes(e.code)); - - const ancestorEntityCodes = ancestorEntities.map(e => e.code); - return ancestorEntityCodes.reduce((obj: Record, ancestor) => { - obj[ancestor] = getDescendantsOfEntities(hierarchyName, [ancestor], descendantQueryOptions); - return obj; - }, {}); + const { field: queryField, ...restOfQueryOptions } = queryOptions; + const { + field: ancestorField = queryField || 'code', + ...restOfAncestorQueryOptions + } = ancestorQueryOptions; + const ancestorOptions = { + ...restOfQueryOptions, + ...restOfAncestorQueryOptions, + fields: [ancestorField, 'code'], + }; + const descendantOptions = { + field: 'code', + ...queryOptions, + ...descendantQueryOptions, + }; + const ancestorEntities = ancestorOptions.filter?.type + ? getRelationsOfEntities(hierarchyName, entityCodes, ancestorOptions) + : getEntities(hierarchyName, entityCodes, ancestorOptions); + const descendantsGroupedByAncestor: Record = ancestorEntities.reduce( + (obj, ancestor) => { + return { + ...obj, + [ancestor[ancestorField]]: getDescendantsOfEntities( + hierarchyName, + [ancestor.code], + descendantOptions, + ), + }; + }, + {} as Record, + ); + + if (groupBy === 'ancestor') { + return descendantsGroupedByAncestor; + } + const ancestorsGroupedByDescendant = Object.entries(descendantsGroupedByAncestor).reduce( + (obj, [ancestor, descendants]) => { + descendants.forEach(descendant => { + // eslint-disable-next-line no-param-reassign + obj[descendant] = ancestor; + }); + return obj; + }, + {} as Record, + ); + return ancestorsGroupedByDescendant; }, }; }; From 363cf0e24583e4c7cfa09535325a196344b3b899 Mon Sep 17 00:00:00 2001 From: Rohan Port Date: Sat, 24 Sep 2022 16:21:21 +1000 Subject: [PATCH 10/17] MAUI-775: Added unit tests for tongaCovidRawData --- packages/report-server/package.json | 3 +- .../customReport/testCustomReport.test.ts | 2 +- .../tongaCovidRawData.fixtures.ts | 188 ++++++++++++++++ .../customReport/tongaCovidRawData.test.ts | 201 ++++++++++++++++++ yarn.lock | 1 + 5 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts create mode 100644 packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts diff --git a/packages/report-server/package.json b/packages/report-server/package.json index 76c1ead6ea..792e43a372 100644 --- a/packages/report-server/package.json +++ b/packages/report-server/package.json @@ -51,6 +51,7 @@ "devDependencies": { "@types/lodash.isplainobject": "^4.0.6", "@types/lodash.keyby": "^4.6.6", - "@types/lodash.pick": "^4.4.0" + "@types/lodash.pick": "^4.4.0", + "mockdate": "^3.0.5" } } diff --git a/packages/report-server/src/__tests__/reportBuilder/customReport/testCustomReport.test.ts b/packages/report-server/src/__tests__/reportBuilder/customReport/testCustomReport.test.ts index 5af48160b9..8a0d42725c 100644 --- a/packages/report-server/src/__tests__/reportBuilder/customReport/testCustomReport.test.ts +++ b/packages/report-server/src/__tests__/reportBuilder/customReport/testCustomReport.test.ts @@ -10,7 +10,7 @@ import { testCustomReport } from '../../../reportBuilder/customReports/testCusto import { entityApiMock } from '../testUtils'; -describe('buildContext', () => { +describe('testCustomReport', () => { const HIERARCHY = 'test_hierarchy'; const ENTITIES = { test_hierarchy: [ diff --git a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts new file mode 100644 index 0000000000..596ac1ed3b --- /dev/null +++ b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts @@ -0,0 +1,188 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ + +export const HIERARCHY = 'test_hierarchy'; +export const ENTITIES = { + test_hierarchy: [ + { code: 'TO', name: 'Tonga', type: 'country' }, + { + code: 'TO_Island_1', + name: 'Tonga Island 1', + type: 'district', + }, + { + code: 'TO_Island_2', + name: 'Tonga Island 2', + type: 'district', + }, + { + code: 'TO_Village_1', + name: 'Tonga Village 1', + type: 'village', + }, + { + code: 'TO_Village_2', + name: 'Tonga Village 2', + type: 'village', + }, + { + code: 'TO_Village_3', + name: 'Tonga Village 3', + type: 'village', + }, + { + code: 'TO_Individual_1', + name: 'Tonga Individual 1', + type: 'individual', + }, + { + code: 'TO_Individual_2', + name: 'Tonga Individual 2', + type: 'individual', + }, + { + code: 'TO_Individual_3', + name: 'Tonga Individual 3', + type: 'individual', + }, + { + code: 'TO_Individual_4', + name: 'Tonga Individual 4', + type: 'individual', + }, + { + code: 'TO_Individual_5', + name: 'Tonga Individual 5', + type: 'individual', + }, + ], +}; + +export const RELATIONS = { + test_hierarchy: [ + { parent: 'TO', child: 'TO_Island_1' }, + { parent: 'TO', child: 'TO_Island_2' }, + { parent: 'TO_Island_1', child: 'TO_Village_1' }, + { parent: 'TO_Island_1', child: 'TO_Village_2' }, + { parent: 'TO_Island_2', child: 'TO_Village_3' }, + { parent: 'TO_Village_1', child: 'TO_Individual_1' }, + { parent: 'TO_Village_1', child: 'TO_Individual_2' }, + { parent: 'TO_Village_2', child: 'TO_Individual_3' }, + { parent: 'TO_Village_3', child: 'TO_Individual_4' }, + { parent: 'TO_Village_3', child: 'TO_Individual_5' }, + ], +}; + +const C19TRegistrationEvents = [ + { + orgUnit: 'TO_Individual_1', + eventDate: '1920-01-01', + dataValues: { + C19T002: 'Sausages', + C19T003: 'Silly', + C19T004: '2020-01-01', + C19T005: 'Male', + C19T006: '9182487324', + }, + }, + { + orgUnit: 'TO_Individual_2', + eventDate: '1921-11-12', + dataValues: { + C19T002: 'Whiskers', + C19T003: 'Paddington', + C19T004: '1970-01-01', + C19T005: 'Female', + C19T006: '9012837273', + }, + }, + { + orgUnit: 'TO_Individual_4', + eventDate: '1924-12-01', + dataValues: { + C19T002: 'Junior', + C19T003: 'Super', + C19T004: '1990-09-13', + C19T005: 'Other', + C19T006: '08090928', + }, + }, +]; + +const C19TResultsEvents = [ + { + orgUnit: 'TO_Individual_2', + eventDate: '2020-01-01', + dataValues: { + C19T012: 'Flu', + C19T013: 'Basic', + C19T013_a: 7, + C19T015: 'Yes', + C19T015_a: 'No', + C19T016: 'Yes', + C19T042: '19/09/2022', + C19T017: 'Yes', + C19T018: 'Yes', + C19T019: 'No', + C19T022: 'No', + C19T020: 'Yes', + C19T038: 'No', + C19T039: 'No', + C19T044: '13/09/2022', + C19T024: 'Disney Land', + C19T025: 'Car park', + C19T026: 'Good', + C19T027: 'Roomy', + C19T028: 'Soccer pitch', + C19T041: 'What?', + C19T029: 'Nope', + C19T033: 13, + C19T034: 7, + C19T035: 'Bad case', + C19T036: 'No', + C19T037: 'N/A', + C19T043: 'Fully vaccinated', + }, + }, + { + orgUnit: 'TO_Individual_4', + eventDate: '2020-05-19', + dataValues: { + C19T012: 'Covid', + C19T013: 'Antigen', + C19T013_a: 4, + C19T015: 'No', + C19T015_a: 'Yes', + C19T016: 'No', + C19T017: 'No', + C19T018: 'No', + C19T019: 'No', + C19T021: 'None', + C19T022: 'No', + C19T020: 'No', + C19T038: 'No', + C19T039: 'No', + C19T044: 'N/A', + C19T024: 'Space', + C19T025: 'Salvos', + C19T026: 'Not bad', + C19T027: 'Circular', + C19T028: 'Van', + C19T041: 'Is RRT?', + C19T029: 'None', + C19T033: 2, + C19T034: 4, + C19T035: 'Good case', + C19T036: 'No', + C19T037: 'N/A', + C19T043: 'Partially vaccinated', + }, + }, +]; + +export const EVENTS: Record }[]> = { + C19T_Registration: C19TRegistrationEvents, + C19T_Results: C19TResultsEvents, +}; diff --git a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts new file mode 100644 index 0000000000..470bfbba35 --- /dev/null +++ b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts @@ -0,0 +1,201 @@ +/** + * Tupaia + * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd + */ + +import MockDate from 'mockdate'; +import { AccessPolicy } from '@tupaia/access-policy'; +import { ReqContext } from '../../../reportBuilder/context'; + +import { tongaCovidRawData } from '../../../reportBuilder/customReports/tongaCovidRawData'; + +import { entityApiMock } from '../testUtils'; +import { ENTITIES, EVENTS, HIERARCHY, RELATIONS } from './tongaCovidRawData.fixtures'; + +const CURRENT_DATE_STUB = '2020-12-15T00:00:00Z'; + +const fetchFakeEvents = ( + surveyCode: string, + { + organisationUnitCodes, + dataElementCodes, + }: { organisationUnitCodes: string[]; dataElementCodes: string[] }, +) => { + return EVENTS[surveyCode] + .filter(event => organisationUnitCodes.includes(event.orgUnit)) + .map(event => ({ + ...event, + dataValues: Object.fromEntries( + Object.entries(event.dataValues).filter(([dataElementCode]) => + dataElementCodes.includes(dataElementCode), + ), + ), + })); +}; + +jest.mock('@tupaia/aggregator', () => ({ + createAggregator: jest.fn().mockImplementation(() => ({ + fetchEvents: fetchFakeEvents, + })), +})); + +describe('tongaCovidRawData', () => { + const apiMock = entityApiMock(ENTITIES, RELATIONS); + + const reqContext: ReqContext = { + hierarchy: HIERARCHY, + permissionGroup: 'Public', + services: { + entity: apiMock, + } as ReqContext['services'], + accessPolicy: new AccessPolicy({ AU: ['Public'] }), + }; + + beforeAll(() => { + MockDate.set(CURRENT_DATE_STUB); + }); + + afterAll(() => { + MockDate.reset(); + }); + + it('builds the tongaCovidRawData report', async () => { + const expectedValue = { + columns: [ + { key: 'Test ID', title: 'Test ID' }, + { key: 'Surname', title: 'Surname' }, + { key: 'Given Names', title: 'Given Names' }, + { key: 'Date of Birth', title: 'Date of Birth' }, + { key: 'Age', title: 'Age' }, + { key: 'Sex', title: 'Sex' }, + { key: 'Phone No.', title: 'Phone No.' }, + { key: 'Island Group', title: 'Island Group' }, + { key: 'Village Code', title: 'Village Code' }, + { key: 'Address', title: 'Address' }, + { key: 'Date of Test', title: 'Date of Test' }, + { key: 'Result', title: 'Result' }, + { key: 'New Case', title: 'New Case' }, + { key: 'Estimated Recovery Date', title: 'Estimated Recovery Date' }, + { key: 'Test Type', title: 'Test Type' }, + { key: 'CT Value', title: 'CT Value' }, + { key: 'Vaccination Status', title: 'Vaccination Status' }, + { key: 'Outbound Traveller', title: 'Outbound Traveller' }, + { key: 'inbound Traveller', title: 'inbound Traveller' }, + { key: 'Symptomatic', title: 'Symptomatic' }, + { key: 'Date of Symptomatic Onset', title: 'Date of Symptomatic Onset' }, + { key: 'Quarantine', title: 'Quarantine' }, + { key: 'Primary Contact', title: 'Primary Contact' }, + { key: 'Frontliner', title: 'Frontliner' }, + { key: 'Frontliner Type', title: 'Frontliner Type' }, + { key: 'Other', title: 'Other' }, + { key: 'Community Testing', title: 'Community Testing' }, + { key: 'Patient', title: 'Patient' }, + { key: 'Other Reason', title: 'Other Reason' }, + { key: 'Primary Contact Testing Day', title: 'Primary Contact Testing Day' }, + { key: 'Testing Site', title: 'Testing Site' }, + { key: 'Quarantine Facility', title: 'Quarantine Facility' }, + { key: 'Ward Type', title: 'Ward Type' }, + { key: 'Clinic Type', title: 'Clinic Type' }, + { key: 'Health Center', title: 'Health Center' }, + { key: 'RRT Team Name', title: 'RRT Team Name' }, + ], + rows: [ + { + Address: 'Tonga Village 1', + Age: 50, + 'CT Value': 7, + 'Clinic Type': 'Roomy', + 'Community Testing': 'Yes', + 'Date Previous Positive': 'N/A', + 'Date of Birth': '1970-00-01', + 'Date of Symptom Onset': '19/09/2022', + 'Date of Test': '2020-01-01', + 'Estimated Recovery Date': 'Not applicable', + Frontliner: 'No', + 'Given Names': 'Paddington', + 'Health Center': 'Soccer pitch', + 'Inbound Traveller': 'No', + 'Island Group': 'Tonga Island 1', + 'New Case': 'Bad case', + Other: 'No', + 'Other Reason': 'No', + 'Other Results': 7, + 'Other Site': 'Nope', + 'Other Type': 'Basic', + 'Outbound Traveller': 'Yes', + Patient: 'No', + 'Phone No.': '9012837273', + 'Previous Positive': 'No', + 'Primary Contact': 'Yes', + 'Primary Contact Testing Day': '13/09/2022', + Quarantine: 'Yes', + 'Quarantine Facility': 'Car park', + RRT: 'What?', + Result: 13, + Sex: 'Female', + Surname: 'Whiskers', + Symptomatic: 'Yes', + 'Test ID': 'TO_Individual_2', + 'Test Type': 'Flu', + 'Testing Site': 'Disney Land', + 'Vaccination Status': 'Fully vaccinated', + 'Village Code': 'TO_Village_1', + 'Ward Type': 'Good', + eventDate: '2020-01-01', + orgUnit: 'TO_Individual_2', + }, + { + Address: 'Tonga Village 3', + Age: 30, + 'CT Value': 4, + 'Clinic Type': 'Circular', + 'Community Testing': 'No', + 'Date Previous Positive': 'N/A', + 'Date of Birth': '1990-00-13', + 'Date of Test': '2020-05-19', + 'Estimated Recovery Date': 'Not applicable', + Frontliner: 'No', + 'Frontliner Type': 'None', + 'Given Names': 'Super', + 'Health Center': 'Van', + 'Inbound Traveller': 'Yes', + 'Island Group': 'Tonga Island 2', + 'New Case': 'Good case', + Other: 'No', + 'Other Reason': 'No', + 'Other Results': 4, + 'Other Site': 'None', + 'Other Type': 'Antigen', + 'Outbound Traveller': 'No', + Patient: 'No', + 'Phone No.': '08090928', + 'Previous Positive': 'No', + 'Primary Contact': 'No', + 'Primary Contact Testing Day': 'N/A', + Quarantine: 'No', + 'Quarantine Facility': 'Salvos', + RRT: 'Is RRT?', + Result: 2, + Sex: 'Other', + Surname: 'Junior', + Symptomatic: 'No', + 'Test ID': 'TO_Individual_4', + 'Test Type': 'Covid', + 'Testing Site': 'Space', + 'Vaccination Status': 'Partially vaccinated', + 'Village Code': 'TO_Village_3', + 'Ward Type': 'Not bad', + eventDate: '2020-05-19', + orgUnit: 'TO_Individual_4', + }, + ], + }; + + const numberOfFacilitiesInTonga = await tongaCovidRawData(reqContext, { + organisationUnitCodes: ['TO'], + hierarchy: 'test_hierarchy', + }); + + expect(numberOfFacilitiesInTonga).toEqual(expectedValue); + }); +}); diff --git a/yarn.lock b/yarn.lock index 8c65171945..8ed0282e7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5587,6 +5587,7 @@ __metadata: lodash.keyby: ^4.6.0 lodash.pick: ^4.4.0 mathjs: ^9.4.0 + mockdate: ^3.0.5 winston: ^3.2.1 languageName: unknown linkType: soft From 213f1412c08bbdeaecb551139e7de0ad36a4b6ef Mon Sep 17 00:00:00 2001 From: Rohan Port Date: Wed, 28 Sep 2022 16:14:06 +1000 Subject: [PATCH 11/17] MAUI-775: Cleanups and performance improvements --- .../customReports/tongaCovidRawData.ts | 37 +++++-------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts index 68b8e99efe..30b497b8f6 100644 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -3,6 +3,7 @@ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd */ import { format, differenceInYears, addDays, isDate } from 'date-fns'; +import keyBy from 'lodash.keyby'; import { createAggregator } from '@tupaia/aggregator'; import { ReportServerAggregator } from '../../aggregator'; import { FetchReportQuery, Event } from '../../types'; @@ -34,11 +35,7 @@ const getRelationships = (reqContext: ReqContext, options: RelationshipsOptions) ) as Promise>; }; -const fetchVillagesAndIslands = async ( - reqContext: ReqContext, - hierarchy: string, - entityCodes: string[], -) => { +const fetchEntities = async (reqContext: ReqContext, hierarchy: string, entityCodes: string[]) => { const villageOptions: RelationshipsOptions = { hierarchy, entityCodes, @@ -58,6 +55,7 @@ const fetchVillagesAndIslands = async ( [villageOptions, islandOptions].map(options => getRelationships(reqContext, options)), ); + const individualCodes = Object.keys(villageCodeByIndividualCodes); const villageCodes = new Set(); Object.values(villageCodeByIndividualCodes).forEach((code: string) => { villageCodes.add(code); @@ -87,6 +85,7 @@ const fetchVillagesAndIslands = async ( }); return { + individualCodes, villageByIndividual: individualCodeByVillageNameAndCode, islandByIndividual: islandNameByIndividualCodes, }; @@ -130,11 +129,11 @@ const fetchEvents = async ( }; const combineAndFlatten = (registrationEvents: Event[], resultEvents: Event[]) => { + const registrationEventsByOrgUnit = keyBy(registrationEvents, 'orgUnit'); const matchedData: Record[] = resultEvents.map(resultEvent => { const { dataValues: resultDataValues, orgUnit: resultOrgUnit, eventDate } = resultEvent; - const matchingRegistration = registrationEvents.find( - registrationEvent => registrationEvent.orgUnit === resultEvent.orgUnit, - ); + const matchingRegistration = registrationEventsByOrgUnit[resultEvent.orgUnit]; + if (!matchingRegistration) { return { orgUnit: resultOrgUnit, @@ -259,10 +258,10 @@ const addVillageAndIsland = ( export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchReportQuery) => { const { organisationUnitCodes: entityCodes, hierarchy, startDate, endDate, period } = query; - const individualCodes = await reqContext.services.entity.getDescendantsOfEntities( + const { individualCodes, villageByIndividual, islandByIndividual } = await fetchEntities( + reqContext, hierarchy, entityCodes, - { field: 'code', filter: { type: 'individual' } }, ); const { registrationEvents, resultsEvents } = await fetchEvents( @@ -275,23 +274,7 @@ export const tongaCovidRawData = async (reqContext: ReqContext, query: FetchRepo ); const builtEvents: Record[] = combineAndFlatten(registrationEvents, resultsEvents); - const rows = builtEvents - .map(rowData => parseRowData(rowData)) - .sort((row, nextRow) => { - if (row['Test ID'] === nextRow['Test ID']) { - return 0; - } - return 1; - }) - .filter((row, index) => index < 20); - - const individualsInRows = Array.from(new Set(rows.map(row => row.orgUnit))); - - const { villageByIndividual, islandByIndividual } = await fetchVillagesAndIslands( - reqContext, - hierarchy, - individualsInRows, - ); + const rows = builtEvents.map(rowData => parseRowData(rowData)); const rowsWithVillageAndIsland = addVillageAndIsland( rows, From 505c0115204d30f5a3d1061e0eb3e5b50c26dae5 Mon Sep 17 00:00:00 2001 From: Rohan Port Date: Wed, 28 Sep 2022 17:13:00 +1000 Subject: [PATCH 12/17] MAUI-775: Convert binary questions to yes/no --- .../tongaCovidRawData.fixtures.ts | 44 ++-- .../customReport/tongaCovidRawData.test.ts | 4 +- .../customReports/data/tongaCovidRawData.json | 203 +++++++++++------- .../customReports/tongaCovidRawData.ts | 91 ++++---- 4 files changed, 191 insertions(+), 151 deletions(-) diff --git a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts index 596ac1ed3b..b909ec74e3 100644 --- a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts +++ b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.fixtures.ts @@ -119,17 +119,17 @@ const C19TResultsEvents = [ C19T012: 'Flu', C19T013: 'Basic', C19T013_a: 7, - C19T015: 'Yes', - C19T015_a: 'No', - C19T016: 'Yes', + C19T015: 1, + C19T015_a: 0, + C19T016: 1, C19T042: '19/09/2022', - C19T017: 'Yes', - C19T018: 'Yes', - C19T019: 'No', + C19T017: 1, + C19T018: 1, + C19T019: 0, C19T022: 'No', - C19T020: 'Yes', - C19T038: 'No', - C19T039: 'No', + C19T020: 1, + C19T038: 0, + C19T039: 0, C19T044: '13/09/2022', C19T024: 'Disney Land', C19T025: 'Car park', @@ -140,8 +140,8 @@ const C19TResultsEvents = [ C19T029: 'Nope', C19T033: 13, C19T034: 7, - C19T035: 'Bad case', - C19T036: 'No', + C19T035: 1, + C19T036: 0, C19T037: 'N/A', C19T043: 'Fully vaccinated', }, @@ -153,17 +153,17 @@ const C19TResultsEvents = [ C19T012: 'Covid', C19T013: 'Antigen', C19T013_a: 4, - C19T015: 'No', - C19T015_a: 'Yes', - C19T016: 'No', - C19T017: 'No', - C19T018: 'No', - C19T019: 'No', + C19T015: 0, + C19T015_a: 1, + C19T016: 0, + C19T017: 0, + C19T018: 0, + C19T019: 0, C19T021: 'None', C19T022: 'No', - C19T020: 'No', - C19T038: 'No', - C19T039: 'No', + C19T020: 0, + C19T038: 0, + C19T039: 0, C19T044: 'N/A', C19T024: 'Space', C19T025: 'Salvos', @@ -174,8 +174,8 @@ const C19TResultsEvents = [ C19T029: 'None', C19T033: 2, C19T034: 4, - C19T035: 'Good case', - C19T036: 'No', + C19T035: 0, + C19T036: 0, C19T037: 'N/A', C19T043: 'Partially vaccinated', }, diff --git a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts index 470bfbba35..cd9b361cf0 100644 --- a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts +++ b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts @@ -116,7 +116,7 @@ describe('tongaCovidRawData', () => { 'Health Center': 'Soccer pitch', 'Inbound Traveller': 'No', 'Island Group': 'Tonga Island 1', - 'New Case': 'Bad case', + 'New Case': 'Yes', Other: 'No', 'Other Reason': 'No', 'Other Results': 7, @@ -160,7 +160,7 @@ describe('tongaCovidRawData', () => { 'Health Center': 'Van', 'Inbound Traveller': 'Yes', 'Island Group': 'Tonga Island 2', - 'New Case': 'Good case', + 'New Case': 'No', Other: 'No', 'Other Reason': 'No', 'Other Results': 4, diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json index a456d570a4..40aa1b2002 100644 --- a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json +++ b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json @@ -1,82 +1,125 @@ { - "C19T_Registration": { - "dataElementCodes": ["C19T002","C19T003","C19T004","C19T005","C19T006"] - }, - "C19T_Results": { - "dataElementCodes": ["C19T012","C19T013","C19T013_a","C19T015","C19T015_a","C19T016","C19T042","C19T017","C19T018","C19T019","C19T021","C19T022","C19T020","C19T038","C19T039","C19T044","C19T024","C19T025","C19T026","C19T027","C19T028","C19T041","C19T029","C19T033","C19T034","C19T035","C19T036","C19T037","C19T043"] - }, - "codesToNames": { - "C19T002": "Surname", - "C19T003": "Given Names", - "C19T004": "dob", - "C19T005": "Sex", - "C19T006": "Phone No.", - "C19T012": "Test Type", - "C19T013": "Other Type", - "C19T013_a":"CT Value", - "C19T015": "Outbound Traveller", - "C19T015_a": "Inbound Traveller", - "C19T016": "Symptomatic", - "C19T042": "Date of Symptom Onset", - "C19T017": "Quarantine", - "C19T018": "Primary Contact", - "C19T019": "Frontliner", - "C19T021": "Frontliner Type", - "C19T022": "Other", - "C19T020": "Community Testing", - "C19T038": "Patient", - "C19T039": "Other Reason", - "C19T044": "Primary Contact Testing Day", - "C19T024": "Testing Site", - "C19T025": "Quarantine Facility", - "C19T026": "Ward Type", - "C19T027": "Clinic Type", - "C19T028": "Health Center", - "C19T041": "RRT", - "C19T029": "Other Site", - "C19T033": "Result", - "C19T034": "Other Results", - "C19T035": "New Case", - "C19T036": "Previous Positive", - "C19T037": "Date Previous Positive", - "C19T043": "Vaccination Status" - }, - "columns": [ - "Test ID", - "Surname", - "Given Names", - "Date of Birth", - "Age", - "Sex", - "Phone No.", - "Island Group", - "Village Code", - "Address", - "Date of Test", - "Result", - "New Case", - "Estimated Recovery Date", - "Test Type", - "CT Value", - "Vaccination Status", - "Outbound Traveller", - "inbound Traveller", - "Symptomatic", - "Date of Symptomatic Onset", - "Quarantine", - "Primary Contact", - "Frontliner", - "Frontliner Type", - "Other", - "Community Testing", - "Patient", - "Other Reason", - "Primary Contact Testing Day", - "Testing Site", - "Quarantine Facility", - "Ward Type", - "Clinic Type", - "Health Center", - "RRT Team Name" + "C19T_Registration": { + "dataElementCodes": ["C19T002", "C19T003", "C19T004", "C19T005", "C19T006"] + }, + "C19T_Results": { + "dataElementCodes": [ + "C19T012", + "C19T013", + "C19T013_a", + "C19T015", + "C19T015_a", + "C19T016", + "C19T042", + "C19T017", + "C19T018", + "C19T019", + "C19T021", + "C19T022", + "C19T020", + "C19T038", + "C19T039", + "C19T044", + "C19T024", + "C19T025", + "C19T026", + "C19T027", + "C19T028", + "C19T041", + "C19T029", + "C19T033", + "C19T034", + "C19T035", + "C19T036", + "C19T037", + "C19T043" ] -} \ No newline at end of file + }, + "codesToNames": { + "C19T002": "Surname", + "C19T003": "Given Names", + "C19T004": "dob", + "C19T005": "Sex", + "C19T006": "Phone No.", + "C19T012": "Test Type", + "C19T013": "Other Type", + "C19T013_a": "CT Value", + "C19T015": "Outbound Traveller", + "C19T015_a": "Inbound Traveller", + "C19T016": "Symptomatic", + "C19T042": "Date of Symptom Onset", + "C19T017": "Quarantine", + "C19T018": "Primary Contact", + "C19T019": "Frontliner", + "C19T021": "Frontliner Type", + "C19T022": "Other", + "C19T020": "Community Testing", + "C19T038": "Patient", + "C19T039": "Other Reason", + "C19T044": "Primary Contact Testing Day", + "C19T024": "Testing Site", + "C19T025": "Quarantine Facility", + "C19T026": "Ward Type", + "C19T027": "Clinic Type", + "C19T028": "Health Center", + "C19T041": "RRT", + "C19T029": "Other Site", + "C19T033": "Result", + "C19T034": "Other Results", + "C19T035": "New Case", + "C19T036": "Previous Positive", + "C19T037": "Date Previous Positive", + "C19T043": "Vaccination Status" + }, + "columns": [ + "Test ID", + "Surname", + "Given Names", + "Date of Birth", + "Age", + "Sex", + "Phone No.", + "Island Group", + "Village Code", + "Address", + "Date of Test", + "Result", + "New Case", + "Estimated Recovery Date", + "Test Type", + "CT Value", + "Vaccination Status", + "Outbound Traveller", + "inbound Traveller", + "Symptomatic", + "Date of Symptomatic Onset", + "Quarantine", + "Primary Contact", + "Frontliner", + "Frontliner Type", + "Other", + "Community Testing", + "Patient", + "Other Reason", + "Primary Contact Testing Day", + "Testing Site", + "Quarantine Facility", + "Ward Type", + "Clinic Type", + "Health Center", + "RRT Team Name" + ], + "binaryAndCheckboxQuestions": [ + "C19T016", + "C19T017", + "C19T018", + "C19T019", + "C19T020", + "C19T038", + "C19T015", + "C19T015_a", + "C19T039", + "C19T035", + "C19T036" + ] +} diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts index 30b497b8f6..1268f9f5d3 100644 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -18,11 +18,6 @@ interface RelationshipsOptions { descendantOptions?: any; } -interface Options { - programCode: string; - dataElementCodes: string[]; -} - const getRelationships = (reqContext: ReqContext, options: RelationshipsOptions) => { const { hierarchy, entityCodes, queryOptions, ancestorOptions, descendantOptions } = options; return reqContext.services.entity.getRelationshipsOfEntities( @@ -99,59 +94,54 @@ const fetchEvents = async ( endDate?: string, period?: string, ) => { - const registrationOptions = { - programCode: 'C19T_Registration', - dataElementCodes: SURVEYS.C19T_Registration.dataElementCodes, - }; - const resultsOptions = { - programCode: 'C19T_Results', - dataElementCodes: SURVEYS.C19T_Results.dataElementCodes, - }; - const aggregator = new ReportServerAggregator(createAggregator(undefined, reqContext)); - const fetch = async (options: Options) => { - const { programCode, dataElementCodes } = options; - return aggregator.fetchEvents( - programCode, - undefined, - entityCodes, - hierarchy, - { startDate, endDate, period }, - dataElementCodes, - ); - }; - - const [registrationEvents, resultsEvents] = await Promise.all( - [registrationOptions, resultsOptions].map(fetch), + const resultEventsPromise = aggregator.fetchEvents( + 'C19T_Results', + undefined, + entityCodes, + hierarchy, + { startDate, endDate, period }, + SURVEYS.C19T_Results.dataElementCodes, ); + const registrationEventsPromise = aggregator.fetchEvents( + 'C19T_Registration', + undefined, + entityCodes, + hierarchy, + {}, + SURVEYS.C19T_Registration.dataElementCodes, + ); + + const [registrationEvents, resultsEvents] = await Promise.all([ + registrationEventsPromise, + resultEventsPromise, + ]); return { registrationEvents, resultsEvents }; }; const combineAndFlatten = (registrationEvents: Event[], resultEvents: Event[]) => { const registrationEventsByOrgUnit = keyBy(registrationEvents, 'orgUnit'); - const matchedData: Record[] = resultEvents.map(resultEvent => { - const { dataValues: resultDataValues, orgUnit: resultOrgUnit, eventDate } = resultEvent; - const matchingRegistration = registrationEventsByOrgUnit[resultEvent.orgUnit]; + const combinedEvents: Record[] = []; + resultEvents.forEach(resultEvent => { + const { dataValues: resultDataValues, orgUnit, eventDate } = resultEvent; + const matchingRegistration = registrationEventsByOrgUnit[orgUnit]; if (!matchingRegistration) { - return { - orgUnit: resultOrgUnit, - eventDate, - ...resultDataValues, - }; + return; } - const { dataValues: registrationDataValues, orgUnit } = matchingRegistration; - return { + const { dataValues: registrationDataValues } = matchingRegistration; + + combinedEvents.push({ orgUnit, eventDate, ...registrationDataValues, ...resultDataValues, - }; + }); }); const now = new Date(); - const dataWithUpdatesAndAddOns = matchedData.map(event => { + const dataWithUpdatesAndAddOns = combinedEvents.map(event => { const { orgUnit, C19T033: result, @@ -201,9 +191,17 @@ const getEstimatedRecoveryDate = ( return isDate(recoveryDate) && format(recoveryDate, 'yyyy-mm-dd'); }; +const binaryToYesNo = (value: unknown) => { + if (value === undefined) { + return undefined; + } + + return value === 1 ? 'Yes' : 'No'; +}; + const parseRowData = (rowData: Record) => { const formattedRow: Record = {}; - const { codesToNames } = SURVEYS; + const { codesToNames, binaryAndCheckboxQuestions } = SURVEYS; Object.keys(rowData).forEach(fieldKey => { switch (fieldKey) { case 'dateSpecimenCollected': { @@ -225,12 +223,11 @@ const parseRowData = (rowData: Record) => { formattedRow[fieldKey] = rowData[fieldKey]; break; default: { - const name = codesToNames[fieldKey as keyof typeof codesToNames]; - if (!name) { - formattedRow[fieldKey] = rowData[fieldKey]; - } else { - formattedRow[name] = rowData[fieldKey]; - } + const name = codesToNames[fieldKey as keyof typeof codesToNames] || fieldKey; + const value = binaryAndCheckboxQuestions.includes(fieldKey) + ? binaryToYesNo(rowData[fieldKey]) + : rowData[fieldKey]; + formattedRow[name] = value; } } }); From ede02982b187650f59626958606d684dc8883c62 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Mon, 3 Oct 2022 10:44:23 +0200 Subject: [PATCH 13/17] Add checkbox property to registration object --- .../reportBuilder/customReports/data/tongaCovidRawData.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json index 40aa1b2002..e5b0b61e5f 100644 --- a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json +++ b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json @@ -1,6 +1,7 @@ { "C19T_Registration": { - "dataElementCodes": ["C19T002", "C19T003", "C19T004", "C19T005", "C19T006"] + "dataElementCodes": ["C19T002", "C19T003", "C19T004", "C19T005", "C19T006"], + "binaryAndCheckboxQuestions": [] }, "C19T_Results": { "dataElementCodes": [ From 5c86dc7478ce81da3d9b34d2a0c67a5631c5c540 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Fri, 21 Oct 2022 06:45:58 +0200 Subject: [PATCH 14/17] Fix date formatting --- .../customReports/data/tongaCovidRawData.json | 4 ++-- .../src/reportBuilder/customReports/tongaCovidRawData.ts | 6 +++--- yarn.lock | 9 +++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json index e5b0b61e5f..bbd2dd4c0d 100644 --- a/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json +++ b/packages/report-server/src/reportBuilder/customReports/data/tongaCovidRawData.json @@ -48,7 +48,7 @@ "C19T015": "Outbound Traveller", "C19T015_a": "Inbound Traveller", "C19T016": "Symptomatic", - "C19T042": "Date of Symptom Onset", + "C19T042": "Date of Symptomatic Onset", "C19T017": "Quarantine", "C19T018": "Primary Contact", "C19T019": "Frontliner", @@ -63,7 +63,7 @@ "C19T026": "Ward Type", "C19T027": "Clinic Type", "C19T028": "Health Center", - "C19T041": "RRT", + "C19T041": "RRT Team Name", "C19T029": "Other Site", "C19T033": "Result", "C19T034": "Other Results", diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts index 1268f9f5d3..9d868c2daa 100644 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -184,11 +184,11 @@ const getEstimatedRecoveryDate = ( if (onsetDate) { const recoveryDate = addDays(new Date(onsetDate), 13); - return isDate(recoveryDate) && format(recoveryDate, 'yyy-mm-dd'); + return isDate(recoveryDate) && format(recoveryDate, 'yyyy-MM-dd'); } const recoveryDate = addDays(new Date(collectionDate), 13); - return isDate(recoveryDate) && format(recoveryDate, 'yyyy-mm-dd'); + return isDate(recoveryDate) && format(recoveryDate, 'yyyy-MM-dd'); }; const binaryToYesNo = (value: unknown) => { @@ -213,7 +213,7 @@ const parseRowData = (rowData: Record) => { formattedRow['Date of Birth'] = 'Unknown'; } const rawDate = new Date(rowData[fieldKey]); - const dobValue = isDate(rawDate) && format(rawDate, 'yyyy-mm-dd'); + const dobValue = isDate(rawDate) && format(rawDate, 'yyyy-MM-dd'); formattedRow['Date of Birth'] = dobValue; break; } diff --git a/yarn.lock b/yarn.lock index 5df23253bf..292dbf0dd8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5581,12 +5581,14 @@ __metadata: api-error-handler: ^1.0.0 body-parser: ^1.18.3 cors: ^2.8.5 + date-fns: ^2.29.3 dotenv: ^8.2.0 express: ^4.16.2 lodash.isplainobject: ^4.0.6 lodash.keyby: ^4.6.0 lodash.pick: ^4.4.0 mathjs: ^9.4.0 + mockdate: ^3.0.5 moment: ^2.24.0 winston: ^3.2.1 languageName: unknown @@ -12640,6 +12642,13 @@ __metadata: languageName: node linkType: hard +"date-fns@npm:^2.29.3": + version: 2.29.3 + resolution: "date-fns@npm:2.29.3" + checksum: e01cf5b62af04e05dfff921bb9c9933310ed0e1ae9a81eb8653452e64dc841acf7f6e01e1a5ae5644d0337e9a7f936175fd2cb6819dc122fdd9c5e86c56be484 + languageName: node + linkType: hard + "dayjs@npm:^1.8.15": version: 1.9.6 resolution: "dayjs@npm:1.9.6" From 609629898c0ca024bca980410f5422ad4cf28544 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Tue, 22 Nov 2022 08:56:48 +0100 Subject: [PATCH 15/17] Fix test --- .../customReport/tongaCovidRawData.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts index cd9b361cf0..eb6c53eb45 100644 --- a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts +++ b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts @@ -107,8 +107,8 @@ describe('tongaCovidRawData', () => { 'Clinic Type': 'Roomy', 'Community Testing': 'Yes', 'Date Previous Positive': 'N/A', - 'Date of Birth': '1970-00-01', - 'Date of Symptom Onset': '19/09/2022', + 'Date of Birth': '1970-01-01', + 'Date of Symptomatic Onset': '19/09/2022', 'Date of Test': '2020-01-01', 'Estimated Recovery Date': 'Not applicable', Frontliner: 'No', @@ -130,7 +130,7 @@ describe('tongaCovidRawData', () => { 'Primary Contact Testing Day': '13/09/2022', Quarantine: 'Yes', 'Quarantine Facility': 'Car park', - RRT: 'What?', + 'RRT Team Name': 'What?', Result: 13, Sex: 'Female', Surname: 'Whiskers', @@ -151,7 +151,7 @@ describe('tongaCovidRawData', () => { 'Clinic Type': 'Circular', 'Community Testing': 'No', 'Date Previous Positive': 'N/A', - 'Date of Birth': '1990-00-13', + 'Date of Birth': '1990-09-13', 'Date of Test': '2020-05-19', 'Estimated Recovery Date': 'Not applicable', Frontliner: 'No', @@ -174,7 +174,7 @@ describe('tongaCovidRawData', () => { 'Primary Contact Testing Day': 'N/A', Quarantine: 'No', 'Quarantine Facility': 'Salvos', - RRT: 'Is RRT?', + 'RRT Team Name': 'Is RRT?', Result: 2, Sex: 'Other', Surname: 'Junior', From 43b641c96adb5d09753985bd9884e0a56455faee Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Thu, 24 Nov 2022 08:39:59 +0100 Subject: [PATCH 16/17] Calculate age from test date --- .../src/reportBuilder/customReports/tongaCovidRawData.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts index 9d868c2daa..a18a32a473 100644 --- a/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts +++ b/packages/report-server/src/reportBuilder/customReports/tongaCovidRawData.ts @@ -140,7 +140,7 @@ const combineAndFlatten = (registrationEvents: Event[], resultEvents: Event[]) = ...resultDataValues, }); }); - const now = new Date(); + const dataWithUpdatesAndAddOns = combinedEvents.map(event => { const { orgUnit, @@ -149,8 +149,8 @@ const combineAndFlatten = (registrationEvents: Event[], resultEvents: Event[]) = C19T004: dob, eventDate: dateSpecimenCollected, } = event; - - const age = getAge(dob, now); + const testDate = new Date(dateSpecimenCollected); + const age = getAge(dob, testDate); return { 'Test ID': orgUnit, 'Estimated Recovery Date': getEstimatedRecoveryDate(result, dateSpecimenCollected, onsetDate), From 138f5d2002ac8a79cf485465af00c4c856c48f97 Mon Sep 17 00:00:00 2001 From: Chris Pollard Date: Thu, 24 Nov 2022 09:25:35 +0100 Subject: [PATCH 17/17] Update tongaCovidRawData.test.ts --- .../reportBuilder/customReport/tongaCovidRawData.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts index eb6c53eb45..a45f26778e 100644 --- a/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts +++ b/packages/report-server/src/__tests__/reportBuilder/customReport/tongaCovidRawData.test.ts @@ -146,7 +146,7 @@ describe('tongaCovidRawData', () => { }, { Address: 'Tonga Village 3', - Age: 30, + Age: 29, 'CT Value': 4, 'Clinic Type': 'Circular', 'Community Testing': 'No',