From 4ec73566bcf6f6da588c801f4123d9ad3d9fa9b0 Mon Sep 17 00:00:00 2001 From: Graham Lee Date: Fri, 6 May 2022 15:40:40 +0100 Subject: [PATCH] Remove age range from mongoose types; fix model tests #2670 Controller tests still fail at this point. --- .../data-service/src/controllers/case.ts | 22 +--- data-serving/data-service/src/model/case.ts | 6 +- .../data-service/src/model/demographics.ts | 47 +++++--- data-serving/data-service/src/util/case.ts | 15 +-- .../test/model/demographics.test.ts | 48 --------- .../data-service/test/util/case.test.ts | 100 ++++++++++++------ 6 files changed, 116 insertions(+), 122 deletions(-) diff --git a/data-serving/data-service/src/controllers/case.ts b/data-serving/data-service/src/controllers/case.ts index e3bb64da1..27443a542 100644 --- a/data-serving/data-service/src/controllers/case.ts +++ b/data-serving/data-service/src/controllers/case.ts @@ -1,5 +1,6 @@ import { Case, + caseAgeRange, CaseDocument, CaseDTO, caseWithDenormalisedConfirmationDate, @@ -67,26 +68,13 @@ const caseFromDTO = async (receivedCase: CaseDTO) => { const dtoFromCase = async (storedCase: LeanDocument) => { let dto = (storedCase as unknown) as CaseDTO; - if ( - storedCase.demographics && - storedCase.demographics.ageBuckets && - storedCase.demographics.ageBuckets.length > 0 - ) { - const ageBuckets = await Promise.all( - storedCase.demographics.ageBuckets.map((bucketId) => { - return AgeBucket.findById(bucketId).lean(); - }), - ); - const minimumAge = Math.min(...ageBuckets.map((b) => b!.start)); - const maximumAge = Math.max(...ageBuckets.map((b) => b!.end)); + const ageRange = await caseAgeRange(storedCase); + if (ageRange) { dto = { ...dto, demographics: { ...dto.demographics!, - ageRange: { - start: minimumAge, - end: maximumAge, - }, + ageRange, }, }; // although the type system can't see it, there's an ageBuckets property on the demographics DTO now @@ -265,7 +253,7 @@ export class CasesController { while (doc != null) { delete doc.restrictedNotes; delete doc.notes; - const normalizedDoc = denormalizeFields(doc); + const normalizedDoc = await denormalizeFields(doc); if (!doc.hasOwnProperty('SGTF')) { normalizedDoc.SGTF = 'NA'; } diff --git a/data-serving/data-service/src/model/case.ts b/data-serving/data-service/src/model/case.ts index bddbb2bac..5bd5b6249 100644 --- a/data-serving/data-service/src/model/case.ts +++ b/data-serving/data-service/src/model/case.ts @@ -1,5 +1,5 @@ import { CaseReferenceDocument, caseReferenceSchema } from './case-reference'; -import { DemographicsDocument, DemographicsDTO, demographicsSchema } from './demographics'; +import { demographicsAgeRange, DemographicsDocument, DemographicsDTO, demographicsSchema } from './demographics'; import { EventDocument, eventSchema } from './event'; import { GenomeSequenceDocument, @@ -204,3 +204,7 @@ export const RestrictedCase = mongoose.model( 'RestrictedCase', caseSchema, ); + +export const caseAgeRange = async (aCase: LeanDocument) => { + return await demographicsAgeRange(aCase.demographics); +}; diff --git a/data-serving/data-service/src/model/demographics.ts b/data-serving/data-service/src/model/demographics.ts index 83a355256..9f39ae574 100644 --- a/data-serving/data-service/src/model/demographics.ts +++ b/data-serving/data-service/src/model/demographics.ts @@ -1,6 +1,7 @@ import { Range } from './range'; -import mongoose from 'mongoose'; +import mongoose, { LeanDocument } from 'mongoose'; import { ObjectId } from 'mongodb'; +import { AgeBucket } from './age-bucket'; /* * There are separate types for demographics for data storage (the mongoose document) and @@ -23,19 +24,6 @@ export const demographicsSchema = new mongoose.Schema( * than one of the buckets we use. */ ageBuckets: [{ type: mongoose.Schema.Types.ObjectId, ref: 'ageBuckets' }], - ageRange: { - start: { - type: Number, - min: 0, - max: 120, - }, - end: { - type: Number, - min: 0, - max: 120, - }, - _id: false, - }, gender: String, occupation: String, nationalities: [String], @@ -44,15 +32,18 @@ export const demographicsSchema = new mongoose.Schema( { _id: false }, ); -export type DemographicsDTO = { - ageRange?: Range; +type DemographicsCommonFields = { gender: string; occupation: string; nationalities: [string]; ethnicity: string; +}; + +export type DemographicsDTO = DemographicsCommonFields & { + ageRange?: Range; } -export type DemographicsDocument = mongoose.Document & DemographicsDTO & { +export type DemographicsDocument = mongoose.Document & DemographicsCommonFields & { ageBuckets: ObjectId[]; }; @@ -60,3 +51,25 @@ export const Demographics = mongoose.model( 'Demographics', demographicsSchema, ); + +export const demographicsAgeRange = async (demographics: LeanDocument) => { + if ( + demographics && + demographics.ageBuckets && + demographics.ageBuckets.length > 0 + ) { + const ageBuckets = await Promise.all( + demographics.ageBuckets.map((bucketId) => { + return AgeBucket.findById(bucketId).lean(); + }), + ); + const minimumAge = Math.min(...ageBuckets.map((b) => b!.start)); + const maximumAge = Math.max(...ageBuckets.map((b) => b!.end)); + return { + start: minimumAge, + end: maximumAge, + }; + } else { + return undefined; + } +}; diff --git a/data-serving/data-service/src/util/case.ts b/data-serving/data-service/src/util/case.ts index b481ec708..49acc2dad 100644 --- a/data-serving/data-service/src/util/case.ts +++ b/data-serving/data-service/src/util/case.ts @@ -1,6 +1,6 @@ import { CaseDocument, CaseDTO } from '../model/case'; import { CaseReferenceDocument } from '../model/case-reference'; -import { DemographicsDocument } from '../model/demographics'; +import { demographicsAgeRange, DemographicsDocument } from '../model/demographics'; import { EventDocument } from '../model/event'; import { LocationDocument } from '../model/location'; import { PathogenDocument } from '../model/pathogen'; @@ -170,11 +170,11 @@ export const removeBlankHeader = (headers: string[]): string[] => { return headers; }; -export const denormalizeFields = (doc: CaseDocument): Partial => { +export const denormalizeFields = async (doc: CaseDocument): Promise> => { const caseReferenceFields = denormalizeCaseReferenceFields( doc.caseReference, ); - const demographicsFields = denormalizeDemographicsFields(doc.demographics); + const demographicsFields = await denormalizeDemographicsFields(doc.demographics); const eventFields = denormalizeEventsFields(doc.events); const locationFields = denormalizeLocationFields(doc.location); const pathogenFields = denormalizePathogenFields(doc.pathogens); @@ -259,12 +259,13 @@ function denormalizeCaseReferenceFields( return denormalizedData; } -function denormalizeDemographicsFields( +async function denormalizeDemographicsFields( doc: DemographicsDocument, -): Record { +): Promise> { const denormalizedData: Record = {}; - denormalizedData['demographics.ageRange.end'] = doc.ageRange?.end || ''; - denormalizedData['demographics.ageRange.start'] = doc.ageRange?.start || ''; + const ageRange = await demographicsAgeRange(doc); + denormalizedData['demographics.ageRange.end'] = ageRange?.end || ''; + denormalizedData['demographics.ageRange.start'] = ageRange?.start || ''; denormalizedData['demographics.ethnicity'] = doc.ethnicity || ''; denormalizedData['demographics.gender'] = doc.gender || ''; const nationalities = diff --git a/data-serving/data-service/test/model/demographics.test.ts b/data-serving/data-service/test/model/demographics.test.ts index 9a7de3756..5b3b8f17a 100644 --- a/data-serving/data-service/test/model/demographics.test.ts +++ b/data-serving/data-service/test/model/demographics.test.ts @@ -3,7 +3,6 @@ import { demographicsSchema, } from '../../src/model/demographics'; -import { Error } from 'mongoose'; import fullModel from './data/demographics.full.json'; import minimalModel from './data/demographics.minimal.json'; import mongoose from 'mongoose'; @@ -14,53 +13,6 @@ const Demographics = mongoose.model( ); describe('validate', () => { - it('a start age under 0 is invalid', async () => { - return new Demographics({ - ...fullModel, - ...{ ageRange: { start: -0.1 } }, - }).validate((e) => { - expect(e).not.toBeNull(); - if (e) expect(e.name).toBe(Error.ValidationError.name); - }); - }); - - it('a start age over 120 is invalid', async () => { - return new Demographics({ - ...fullModel, - ...{ ageRange: { start: 121 } }, - }).validate((e) => { - expect(e).not.toBeNull(); - if (e) expect(e.name).toBe(Error.ValidationError.name); - }); - }); - - it('an end age under 0 is invalid', async () => { - return new Demographics({ - ...fullModel, - ...{ ageRange: { end: -2 } }, - }).validate((e) => { - expect(e).not.toBeNull(); - if (e) expect(e.name).toBe(Error.ValidationError.name); - }); - }); - - it('a start age without end is valid', async () => { - return new Demographics({ - ...fullModel, - ...{ ageRange: { start: 85 } }, - }).validate(); - }); - - it('an end age over 120 is invalid', async () => { - return new Demographics({ - ...fullModel, - ...{ ageRange: { end: 120.1 } }, - }).validate((e) => { - expect(e).not.toBeNull(); - if (e) expect(e.name).toBe(Error.ValidationError.name); - }); - }); - it('a minimal demographics document is valid', async () => { return new Demographics(minimalModel).validate(); }); diff --git a/data-serving/data-service/test/util/case.test.ts b/data-serving/data-service/test/util/case.test.ts index b65668039..d4ba535f8 100644 --- a/data-serving/data-service/test/util/case.test.ts +++ b/data-serving/data-service/test/util/case.test.ts @@ -1,6 +1,7 @@ // @ts-nocheck Unable to block-ignore errors ('Property does not exist' in this file) // https://github.com/Microsoft/TypeScript/issues/19573 +import { AgeBucket } from '../../src/model/age-bucket'; import { CaseDocument } from '../model/case'; import { CaseReferenceDocument } from '../model/case-reference'; import { DemographicsDocument } from '../model/demographics'; @@ -22,6 +23,43 @@ import { denormalizeFields, } from '../../src/util/case'; import events from '../model/data/case.events.json'; +import mongoose from 'mongoose'; +import MongoMemoryServer from 'mongodb-memory-server'; + + +let mongoServer: MongoMemoryServer; + +async function createAgeBuckets() { + await new AgeBucket({ + start: 0, + end: 0, + }).save(); + for (let start = 1; start <= 116; start += 5) { + const end = start + 4; + await new AgeBucket({ + start, + end, + }).save(); + } +} + +beforeAll(async () => { + mongoServer = new MongoMemoryServer(); + const mongoURL = process.env.MONGO_URL; + await mongoose.connect(mongoURL, { + useCreateIndex: true, + useNewUrlParser: true, + useUnifiedTopology: true, + useFindAndModify: false, + }); + await createAgeBuckets(); +}); + +afterAll(async () => { + await AgeBucket.deleteMany({}); + await mongoose.disconnect(); + return mongoServer.stop(); +}); describe('Case', () => { it('is parsed properly for download', () => { @@ -177,7 +215,7 @@ describe('Case', () => { 'vaccines.3.batch','vaccines.3.date','vaccines.3.sideEffects'] ); }); - it('handles any undefined fields', () => { + it('handles any undefined fields', async () => { const caseDoc = { caseReference: {} as CaseReferenceDocument, demographics: {} as DemographicsDocument, @@ -193,7 +231,7 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['caseReference.sourceId']).toEqual(''); expect(denormalizedCase['caseReference.sourceEntryId']).toEqual(''); @@ -268,7 +306,7 @@ describe('Case', () => { expect(denormalizedCase['vaccines.3.sideEffects']).toEqual(''); expect(denormalizedCase['variantOfConcern']).toEqual(''); }); - it('denormalizes case reference fields', () => { + it('denormalizes case reference fields', async () => { const caseRefDoc = { sourceId: 'a source id', sourceEntryId: 'a source entry id', @@ -296,7 +334,7 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['caseReference.sourceId']).toEqual('a source id'); expect(denormalizedCase['caseReference.sourceEntryId']).toEqual('a source entry id'); @@ -305,12 +343,10 @@ describe('Case', () => { expect(denormalizedCase['caseReference.verificationStatus']).toEqual('UNVERIFIED'); expect(denormalizedCase['caseReference.additionalSources']).toEqual('google.com,ap.org'); }); - it('denormalizes demographics fields', () => { + it('denormalizes demographics fields', async () => { + const anAgeBucket = await AgeBucket.findOne({ start: 41 }); const demographicsDoc = { - ageRange: { - start: 42, - end: 50, - }, + ageBuckets: [anAgeBucket._id], gender: 'Male', occupation: 'Anesthesiologist', nationalities: ['Georgian', 'Azerbaijani'], @@ -332,16 +368,16 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); - expect(denormalizedCase['demographics.ageRange.end']).toEqual(50); - expect(denormalizedCase['demographics.ageRange.start']).toEqual(42); + expect(denormalizedCase['demographics.ageRange.end']).toEqual(45); + expect(denormalizedCase['demographics.ageRange.start']).toEqual(41); expect(denormalizedCase['demographics.ethnicity']).toEqual('Caucasian'); expect(denormalizedCase['demographics.gender']).toEqual('Male'); expect(denormalizedCase['demographics.nationalities']).toEqual('Georgian,Azerbaijani'); expect(denormalizedCase['demographics.occupation']).toEqual('Anesthesiologist'); }); - it('denormalizes events fields', () => { + it('denormalizes events fields', async () => { const consultEvent = { name: 'firstClinicalConsultation', dateRange: { @@ -382,7 +418,7 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['events.firstClinicalConsultation.date']).toEqual(consultEvent.dateRange.end); expect(denormalizedCase['events.onsetSymptoms.date']).toEqual(onsetEvent.dateRange.end); @@ -396,7 +432,7 @@ describe('Case', () => { expect(denormalizedCase['events.icuAdmission.date']).toEqual(''); expect(denormalizedCase['events.icuAdmission.value']).toEqual(''); }); - it('denormalizes location fields', () => { + it('denormalizes location fields', async () => { const locationDoc = { country: 'Georgia', name: 'Tbilisi', @@ -422,7 +458,7 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['location.country']).toEqual(locationDoc.country); expect(denormalizedCase['location.administrativeAreaLevel1']).toEqual(''); @@ -435,7 +471,7 @@ describe('Case', () => { expect(denormalizedCase['location.place']).toEqual(''); expect(denormalizedCase['location.query']).toEqual(''); }); - it('denormalizes pathogen fields', () => { + it('denormalizes pathogen fields', async () => { const bacteriaDoc = { name: 'E. coli', id: '0', @@ -466,11 +502,11 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); const pathogenNames = [bacteriaDoc.name, virusDoc.name, fungiDoc.name].join(','); expect(denormalizedCase['pathogens']).toEqual(pathogenNames); }); - it('denormalizes preexisting conditions fields', () => { + it('denormalizes preexisting conditions fields', async () => { const conditionsDoc = { values: ['Obesity', 'Diabetes'], hasPreexistingConditions: true, @@ -491,11 +527,11 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['preexistingConditions.hasPreexistingConditions']).toEqual(true); expect(denormalizedCase['preexistingConditions.values']).toEqual('Obesity,Diabetes'); }); - it('denormalizes revision metadata fields', () => { + it('denormalizes revision metadata fields', async () => { const revisionDoc = { revisionNumber: 4, creationMetadata: { @@ -525,7 +561,7 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['revisionMetadata.creationMetadata.curator']).toEqual('Joe'); expect(denormalizedCase['revisionMetadata.creationMetadata.date']).toEqual('2020-05-01'); @@ -535,7 +571,7 @@ describe('Case', () => { expect(denormalizedCase['revisionMetadata.editMetadata.notes']).toEqual('removed some information'); expect(denormalizedCase['revisionMetadata.revisionNumber']).toEqual(4); }); - it('denormalizes symptoms fields', () => { + it('denormalizes symptoms fields', async () => { const symptomsDoc = { values: ['Cough', 'Fever'], status: 'current', @@ -555,12 +591,12 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['symptoms.values']).toEqual('Cough,Fever'); expect(denormalizedCase['symptoms.status']).toEqual('current'); }); - it('denormalizes transmission fields', () => { + it('denormalizes transmission fields', async () => { const transmissionDoc = { linkedCaseIds: ['0', '1', '2'], places: ['Tbilisi', 'Baku'], @@ -582,13 +618,13 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['transmission.linkedCaseIds']).toEqual('0,1,2'); expect(denormalizedCase['transmission.places']).toEqual('Tbilisi,Baku'); expect(denormalizedCase['transmission.routes']).toEqual('train,plane'); }); - it('denormalizes travel history fields', () => { + it('denormalizes travel history fields', async () => { const travelHistoryDoc = { travel: [{ dateRange: { @@ -642,7 +678,7 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['travelHistory.travel.dateRange.end']).toEqual('2020-05-03,2020-06-03'); expect(denormalizedCase['travelHistory.travel.dateRange.start']).toEqual('2020-05-01,2020-06-01'); @@ -652,7 +688,7 @@ describe('Case', () => { expect(denormalizedCase['travelHistory.travel.purpose']).toEqual('business,pleasure'); expect(denormalizedCase['travelHistory.traveledPrior30Days']).toEqual(true); }); - it('denormalizes vaccine fields', () => { + it('denormalizes vaccine fields', async () => { const firstVaccineDoc = { name: 'Pfizer', batch: 'TK421', @@ -702,7 +738,7 @@ describe('Case', () => { variant: {} as VariantDocument, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['vaccines.0.batch']).toEqual('TK421'); expect(denormalizedCase['vaccines.0.date']).toEqual('2021-03-01'); expect(denormalizedCase['vaccines.0.name']).toEqual('Pfizer'); @@ -720,7 +756,7 @@ describe('Case', () => { expect(denormalizedCase['vaccines.3.name']).toEqual(''); expect(denormalizedCase['vaccines.3.sideEffects']).toEqual(''); }); - it('denormalizes variant fields', () => { + it('denormalizes variant fields', async () => { const variantDoc = { name: 'Omicron', } as VariantDocument; @@ -740,7 +776,7 @@ describe('Case', () => { variant: variantDoc, } as CaseDocument; - const denormalizedCase = denormalizeFields(caseDoc); + const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['variantOfConcern']).toEqual('Omicron'); });