2670 bucket ages

data-serving/data-service/src/controllers/case.ts

@@ -1,5 +1,6 @@
 import {
     Case,
+    caseAgeRange,
     CaseDocument,
     CaseDTO,
     caseWithDenormalisedConfirmationDate,
@@ -45,6 +46,7 @@ const caseFromDTO = async (receivedCase: CaseDTO) => {
         const allBuckets = await AgeBucket.find({});
         const caseStart = receivedCase.demographics?.ageRange.start;
         const caseEnd = receivedCase.demographics?.ageRange.end;
+        validateCaseAges(caseStart, caseEnd);
         const matchingBucketIDs = allBuckets
             .filter((b) => {
                 const bucketContainsStart =
@@ -67,26 +69,13 @@ const caseFromDTO = async (receivedCase: CaseDTO) => {
 
 const dtoFromCase = async (storedCase: LeanDocument<CaseDocument>) => {
     let dto = (storedCase as unknown) as CaseDTO;
-    if (
-        storedCase.demographics &&
-        storedCase.demographics.ageBuckets &&
-        storedCase.demographics.ageBuckets.length > 0
-    ) {
-        const ageBuckets = await Promise.all(
-            storedCase.demographics.ageBuckets.map((bucketId) => {
-                return AgeBucket.findById(bucketId).lean();
-            }),
-        );
-        const minimumAge = Math.min(...ageBuckets.map((b) => b!.start));
-        const maximumAge = Math.max(...ageBuckets.map((b) => b!.end));
+    const ageRange = await caseAgeRange(storedCase);
+    if (ageRange) {
         dto = {
             ...dto,
             demographics: {
                 ...dto.demographics!,
-                ageRange: {
-                    start: minimumAge,
-                    end: maximumAge,
-                },
+                ageRange,
             },
         };
         // although the type system can't see it, there's an ageBuckets property on the demographics DTO now
@@ -265,7 +254,7 @@ export class CasesController {
                 while (doc != null) {
                     delete doc.restrictedNotes;
                     delete doc.notes;
-                    const normalizedDoc = denormalizeFields(doc);
+                    const normalizedDoc = await denormalizeFields(doc);
                     if (!doc.hasOwnProperty('SGTF')) {
                         normalizedDoc.SGTF = 'NA';
                     }
@@ -428,7 +417,8 @@ export class CasesController {
                 }
             }
             res.status(201).json(result);
-        } catch (err) {
+        } catch (e) {
+            const err = e as Error;
             if (err instanceof GeocodeNotFoundError) {
                 res.status(404).json({
                     message: err.message,
@@ -439,12 +429,12 @@ export class CasesController {
                 err.name === 'ValidationError' ||
                 err instanceof InvalidParamError
             ) {
-                console.error('validation error');
-                console.error(err);
+                logger.error('validation error');
+                logger.error(err);
                 res.status(422).json(err);
                 return;
             }
-            console.error(err);
+            logger.error(err);
             res.status(500).json(err);
             return;
         }
@@ -464,6 +454,15 @@ export class CasesController {
         // sequentially, so if Mongo creates a batch validate method that should be used here.
         for (let index = 0; index < cases.length; index++) {
             const c = cases[index];
+            const ageStart = c.demographics?.ageRange?.start;
+            const ageEnd = c.demographics?.ageRange?.end;
+            try {
+                validateCaseAges(ageStart, ageEnd);
+            } catch (e) {
+                const err = e as Error;
+                errors.push({ index, message: err.message });
+                continue;
+            }
             await new Case(c).validate().catch((e) => {
                 errors.push({ index: index, message: e.message });
             });
@@ -611,10 +610,11 @@ export class CasesController {
         } catch (e) {
             const err = e as Error;
             if (err.name === 'ValidationError') {
+                logger.error(err);
                 res.status(422).json(err);
                 return;
             }
-            logger.warn(err);
+            logger.error(err);
             res.status(500).json(err);
             return;
         }
@@ -808,7 +808,7 @@ export class CasesController {
             await RestrictedCase.deleteMany(casesQuery);
             res.status(204).end();
         } catch (err) {
-            console.error(err);
+            logger.error(err);
             res.status(500).json(err);
         }
     };
@@ -1121,9 +1121,9 @@ export const casesMatchingSearchQuery = (opts: {
 }): any => {
     // set data limit to 10K by default
     const countLimit = opts.limit ? opts.limit : 10000;
-    console.log(`Search query: ${opts.searchQuery}`);
+    logger.info(`Search query: ${opts.searchQuery}`);
     const parsedSearch = parseSearchQuery(opts.searchQuery);
-    console.log(`Parsed search (full text?): ${parsedSearch.fullTextSearch}`);
+    logger.debug(`Parsed search (full text?): ${parsedSearch.fullTextSearch}`);
     const queryOpts = parsedSearch.fullTextSearch
         ? {
               $text: { $search: parsedSearch.fullTextSearch },
@@ -1342,3 +1342,9 @@ export const listOccupations = async (
         return;
     }
 };
+function validateCaseAges(caseStart: number, caseEnd: number) {
+    if (caseStart < 0 || caseEnd < caseStart || caseStart > 120 || caseEnd > 120) {
+        throw new InvalidParamError(`Case validation failed: age range ${caseStart}-${caseEnd} invalid (must be within 0-120)`);
+    }
+}
+

data-serving/data-service/src/model/case.ts

@@ -1,5 +1,5 @@
 import { CaseReferenceDocument, caseReferenceSchema } from './case-reference';
-import { DemographicsDocument, DemographicsDTO, demographicsSchema } from './demographics';
+import { demographicsAgeRange, DemographicsDocument, DemographicsDTO, demographicsSchema } from './demographics';
 import { EventDocument, eventSchema } from './event';
 import {
     GenomeSequenceDocument,
@@ -204,3 +204,7 @@ export const RestrictedCase = mongoose.model<CaseDocument>(
     'RestrictedCase',
     caseSchema,
 );
+
+export const caseAgeRange = async (aCase: LeanDocument<CaseDocument>) => {
+   return await demographicsAgeRange(aCase.demographics);
+};

data-serving/data-service/src/model/demographics.ts

@@ -1,6 +1,7 @@
 import { Range } from './range';
-import mongoose from 'mongoose';
+import mongoose, { LeanDocument } from 'mongoose';
 import { ObjectId } from 'mongodb';
+import { AgeBucket } from './age-bucket';
 
 /*
  * There are separate types for demographics for data storage (the mongoose document) and
@@ -23,19 +24,6 @@ export const demographicsSchema = new mongoose.Schema(
          * than one of the buckets we use.
          */
         ageBuckets: [{ type: mongoose.Schema.Types.ObjectId, ref: 'ageBuckets' }],
-        ageRange: {
-            start: {
-                type: Number,
-                min: 0,
-                max: 120,
-            },
-            end: {
-                type: Number,
-                min: 0,
-                max: 120,
-            },
-            _id: false,
-        },
         gender: String,
         occupation: String,
         nationalities: [String],
@@ -44,19 +32,44 @@ export const demographicsSchema = new mongoose.Schema(
     { _id: false },
 );
 
-export type DemographicsDTO = {
-    ageRange?: Range<number>;
+type DemographicsCommonFields = {
     gender: string;
     occupation: string;
     nationalities: [string];
     ethnicity: string;
+};
+
+export type DemographicsDTO = DemographicsCommonFields & {
+    ageRange?: Range<number>;
 }
 
-export type DemographicsDocument = mongoose.Document & DemographicsDTO & {
+export type DemographicsDocument = mongoose.Document & DemographicsCommonFields & {
     ageBuckets: ObjectId[];
 };
 
 export const Demographics = mongoose.model<DemographicsDocument>(
     'Demographics',
     demographicsSchema,
 );
+
+export const demographicsAgeRange = async (demographics: LeanDocument<DemographicsDocument>) => {
+    if (
+        demographics &&
+        demographics.ageBuckets &&
+        demographics.ageBuckets.length > 0
+    ) {
+        const ageBuckets = await Promise.all(
+            demographics.ageBuckets.map((bucketId) => {
+                return AgeBucket.findById(bucketId).lean();
+            }),
+        );
+        const minimumAge = Math.min(...ageBuckets.map((b) => b!.start));
+        const maximumAge = Math.max(...ageBuckets.map((b) => b!.end));
+        return {
+            start: minimumAge,
+            end: maximumAge,
+        };
+    } else {
+        return undefined;
+    }
+};

data-serving/data-service/src/util/case.ts

@@ -1,6 +1,6 @@
 import { CaseDocument, CaseDTO } from '../model/case';
 import { CaseReferenceDocument } from '../model/case-reference';
-import { DemographicsDocument } from '../model/demographics';
+import { demographicsAgeRange, DemographicsDocument } from '../model/demographics';
 import { EventDocument } from '../model/event';
 import { LocationDocument } from '../model/location';
 import { PathogenDocument } from '../model/pathogen';
@@ -170,11 +170,11 @@ export const removeBlankHeader = (headers: string[]): string[] => {
     return headers;
 };
 
-export const denormalizeFields = (doc: CaseDocument): Partial<CaseDocument> => {
+export const denormalizeFields = async (doc: CaseDocument): Promise<Partial<CaseDocument>> => {
     const caseReferenceFields = denormalizeCaseReferenceFields(
         doc.caseReference,
     );
-    const demographicsFields = denormalizeDemographicsFields(doc.demographics);
+    const demographicsFields = await denormalizeDemographicsFields(doc.demographics);
     const eventFields = denormalizeEventsFields(doc.events);
     const locationFields = denormalizeLocationFields(doc.location);
     const pathogenFields = denormalizePathogenFields(doc.pathogens);
@@ -259,12 +259,13 @@ function denormalizeCaseReferenceFields(
     return denormalizedData;
 }
 
-function denormalizeDemographicsFields(
+async function denormalizeDemographicsFields(
     doc: DemographicsDocument,
-): Record<string, string | number> {
+): Promise<Record<string, string | number>> {
     const denormalizedData: Record<string, string | number> = {};
-    denormalizedData['demographics.ageRange.end'] = doc.ageRange?.end || '';
-    denormalizedData['demographics.ageRange.start'] = doc.ageRange?.start || '';
+    const ageRange = await demographicsAgeRange(doc);
+    denormalizedData['demographics.ageRange.end'] = ageRange?.end || '';
+    denormalizedData['demographics.ageRange.start'] = ageRange?.start || '';
     denormalizedData['demographics.ethnicity'] = doc.ethnicity || '';
     denormalizedData['demographics.gender'] = doc.gender || '';
     const nationalities =

data-serving/data-service/test/model/demographics.test.ts

@@ -3,7 +3,6 @@ import {
     demographicsSchema,
 } from '../../src/model/demographics';
 
-import { Error } from 'mongoose';
 import fullModel from './data/demographics.full.json';
 import minimalModel from './data/demographics.minimal.json';
 import mongoose from 'mongoose';
@@ -14,53 +13,6 @@ const Demographics = mongoose.model<DemographicsDocument>(
 );
 
 describe('validate', () => {
-    it('a start age under 0 is invalid', async () => {
-        return new Demographics({
-            ...fullModel,
-            ...{ ageRange: { start: -0.1 } },
-        }).validate((e) => {
-            expect(e).not.toBeNull();
-            if (e) expect(e.name).toBe(Error.ValidationError.name);
-        });
-    });
-
-    it('a start age over 120 is invalid', async () => {
-        return new Demographics({
-            ...fullModel,
-            ...{ ageRange: { start: 121 } },
-        }).validate((e) => {
-            expect(e).not.toBeNull();
-            if (e) expect(e.name).toBe(Error.ValidationError.name);
-        });
-    });
-
-    it('an end age under 0 is invalid', async () => {
-        return new Demographics({
-            ...fullModel,
-            ...{ ageRange: { end: -2 } },
-        }).validate((e) => {
-            expect(e).not.toBeNull();
-            if (e) expect(e.name).toBe(Error.ValidationError.name);
-        });
-    });
-
-    it('a start age without end is valid', async () => {
-        return new Demographics({
-            ...fullModel,
-            ...{ ageRange: { start: 85 } },
-        }).validate();
-    });
-
-    it('an end age over 120 is invalid', async () => {
-        return new Demographics({
-            ...fullModel,
-            ...{ ageRange: { end: 120.1 } },
-        }).validate((e) => {
-            expect(e).not.toBeNull();
-            if (e) expect(e.name).toBe(Error.ValidationError.name);
-        });
-    });
-
     it('a minimal demographics document is valid', async () => {
         return new Demographics(minimalModel).validate();
     });