Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

2670 bucket ages #2694

Merged
merged 4 commits into from
May 9, 2022
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 31 additions & 25 deletions data-serving/data-service/src/controllers/case.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {
Case,
caseAgeRange,
CaseDocument,
CaseDTO,
caseWithDenormalisedConfirmationDate,
Expand Down Expand Up @@ -45,6 +46,7 @@ const caseFromDTO = async (receivedCase: CaseDTO) => {
const allBuckets = await AgeBucket.find({});
const caseStart = receivedCase.demographics?.ageRange.start;
const caseEnd = receivedCase.demographics?.ageRange.end;
validateCaseAges(caseStart, caseEnd);
const matchingBucketIDs = allBuckets
.filter((b) => {
const bucketContainsStart =
Expand All @@ -67,26 +69,13 @@ const caseFromDTO = async (receivedCase: CaseDTO) => {

const dtoFromCase = async (storedCase: LeanDocument<CaseDocument>) => {
let dto = (storedCase as unknown) as CaseDTO;
if (
storedCase.demographics &&
storedCase.demographics.ageBuckets &&
storedCase.demographics.ageBuckets.length > 0
) {
const ageBuckets = await Promise.all(
storedCase.demographics.ageBuckets.map((bucketId) => {
return AgeBucket.findById(bucketId).lean();
}),
);
const minimumAge = Math.min(...ageBuckets.map((b) => b!.start));
const maximumAge = Math.max(...ageBuckets.map((b) => b!.end));
const ageRange = await caseAgeRange(storedCase);
if (ageRange) {
dto = {
...dto,
demographics: {
...dto.demographics!,
ageRange: {
start: minimumAge,
end: maximumAge,
},
ageRange,
},
};
// although the type system can't see it, there's an ageBuckets property on the demographics DTO now
Expand Down Expand Up @@ -265,7 +254,7 @@ export class CasesController {
while (doc != null) {
delete doc.restrictedNotes;
delete doc.notes;
const normalizedDoc = denormalizeFields(doc);
const normalizedDoc = await denormalizeFields(doc);
if (!doc.hasOwnProperty('SGTF')) {
normalizedDoc.SGTF = 'NA';
}
Expand Down Expand Up @@ -428,7 +417,8 @@ export class CasesController {
}
}
res.status(201).json(result);
} catch (err) {
} catch (e) {
const err = e as Error;
if (err instanceof GeocodeNotFoundError) {
res.status(404).json({
message: err.message,
Expand All @@ -439,12 +429,12 @@ export class CasesController {
err.name === 'ValidationError' ||
err instanceof InvalidParamError
) {
console.error('validation error');
console.error(err);
logger.error('validation error');
logger.error(err);
res.status(422).json(err);
return;
}
console.error(err);
logger.error(err);
res.status(500).json(err);
return;
}
Expand All @@ -464,6 +454,15 @@ export class CasesController {
// sequentially, so if Mongo creates a batch validate method that should be used here.
for (let index = 0; index < cases.length; index++) {
const c = cases[index];
const ageStart = c.demographics?.ageRange?.start;
const ageEnd = c.demographics?.ageRange?.end;
try {
validateCaseAges(ageStart, ageEnd);
} catch (e) {
const err = e as Error;
errors.push({ index, message: err.message });
continue;
}
await new Case(c).validate().catch((e) => {
errors.push({ index: index, message: e.message });
});
Expand Down Expand Up @@ -611,10 +610,11 @@ export class CasesController {
} catch (e) {
const err = e as Error;
if (err.name === 'ValidationError') {
logger.error(err);
res.status(422).json(err);
return;
}
logger.warn(err);
logger.error(err);
res.status(500).json(err);
return;
}
Expand Down Expand Up @@ -808,7 +808,7 @@ export class CasesController {
await RestrictedCase.deleteMany(casesQuery);
res.status(204).end();
} catch (err) {
console.error(err);
logger.error(err);
res.status(500).json(err);
}
};
Expand Down Expand Up @@ -1121,9 +1121,9 @@ export const casesMatchingSearchQuery = (opts: {
}): any => {
// set data limit to 10K by default
const countLimit = opts.limit ? opts.limit : 10000;
console.log(`Search query: ${opts.searchQuery}`);
logger.info(`Search query: ${opts.searchQuery}`);
const parsedSearch = parseSearchQuery(opts.searchQuery);
console.log(`Parsed search (full text?): ${parsedSearch.fullTextSearch}`);
logger.debug(`Parsed search (full text?): ${parsedSearch.fullTextSearch}`);
const queryOpts = parsedSearch.fullTextSearch
? {
$text: { $search: parsedSearch.fullTextSearch },
Expand Down Expand Up @@ -1342,3 +1342,9 @@ export const listOccupations = async (
return;
}
};
function validateCaseAges(caseStart: number, caseEnd: number) {
if (caseStart < 0 || caseEnd < caseStart || caseStart > 120 || caseEnd > 120) {
throw new InvalidParamError(`Case validation failed: age range ${caseStart}-${caseEnd} invalid (must be within 0-120)`);
}
}

6 changes: 5 additions & 1 deletion data-serving/data-service/src/model/case.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { CaseReferenceDocument, caseReferenceSchema } from './case-reference';
import { DemographicsDocument, DemographicsDTO, demographicsSchema } from './demographics';
import { demographicsAgeRange, DemographicsDocument, DemographicsDTO, demographicsSchema } from './demographics';
import { EventDocument, eventSchema } from './event';
import {
GenomeSequenceDocument,
Expand Down Expand Up @@ -204,3 +204,7 @@ export const RestrictedCase = mongoose.model<CaseDocument>(
'RestrictedCase',
caseSchema,
);

export const caseAgeRange = async (aCase: LeanDocument<CaseDocument>) => {
return await demographicsAgeRange(aCase.demographics);
};
47 changes: 30 additions & 17 deletions data-serving/data-service/src/model/demographics.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Range } from './range';
import mongoose from 'mongoose';
import mongoose, { LeanDocument } from 'mongoose';
import { ObjectId } from 'mongodb';
import { AgeBucket } from './age-bucket';

/*
* There are separate types for demographics for data storage (the mongoose document) and
Expand All @@ -23,19 +24,6 @@ export const demographicsSchema = new mongoose.Schema(
* than one of the buckets we use.
*/
ageBuckets: [{ type: mongoose.Schema.Types.ObjectId, ref: 'ageBuckets' }],
ageRange: {
start: {
type: Number,
min: 0,
max: 120,
},
end: {
type: Number,
min: 0,
max: 120,
},
_id: false,
},
gender: String,
occupation: String,
nationalities: [String],
Expand All @@ -44,19 +32,44 @@ export const demographicsSchema = new mongoose.Schema(
{ _id: false },
);

export type DemographicsDTO = {
ageRange?: Range<number>;
type DemographicsCommonFields = {
gender: string;
occupation: string;
nationalities: [string];
ethnicity: string;
};

export type DemographicsDTO = DemographicsCommonFields & {
ageRange?: Range<number>;
}

export type DemographicsDocument = mongoose.Document & DemographicsDTO & {
export type DemographicsDocument = mongoose.Document & DemographicsCommonFields & {
ageBuckets: ObjectId[];
};

export const Demographics = mongoose.model<DemographicsDocument>(
'Demographics',
demographicsSchema,
);

export const demographicsAgeRange = async (demographics: LeanDocument<DemographicsDocument>) => {
if (
demographics &&
demographics.ageBuckets &&
demographics.ageBuckets.length > 0
) {
const ageBuckets = await Promise.all(
demographics.ageBuckets.map((bucketId) => {
return AgeBucket.findById(bucketId).lean();
}),
);
const minimumAge = Math.min(...ageBuckets.map((b) => b!.start));
const maximumAge = Math.max(...ageBuckets.map((b) => b!.end));
return {
start: minimumAge,
end: maximumAge,
};
} else {
return undefined;
}
};
15 changes: 8 additions & 7 deletions data-serving/data-service/src/util/case.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { CaseDocument, CaseDTO } from '../model/case';
import { CaseReferenceDocument } from '../model/case-reference';
import { DemographicsDocument } from '../model/demographics';
import { demographicsAgeRange, DemographicsDocument } from '../model/demographics';
import { EventDocument } from '../model/event';
import { LocationDocument } from '../model/location';
import { PathogenDocument } from '../model/pathogen';
Expand Down Expand Up @@ -170,11 +170,11 @@ export const removeBlankHeader = (headers: string[]): string[] => {
return headers;
};

export const denormalizeFields = (doc: CaseDocument): Partial<CaseDocument> => {
export const denormalizeFields = async (doc: CaseDocument): Promise<Partial<CaseDocument>> => {
const caseReferenceFields = denormalizeCaseReferenceFields(
doc.caseReference,
);
const demographicsFields = denormalizeDemographicsFields(doc.demographics);
const demographicsFields = await denormalizeDemographicsFields(doc.demographics);
const eventFields = denormalizeEventsFields(doc.events);
const locationFields = denormalizeLocationFields(doc.location);
const pathogenFields = denormalizePathogenFields(doc.pathogens);
Expand Down Expand Up @@ -259,12 +259,13 @@ function denormalizeCaseReferenceFields(
return denormalizedData;
}

function denormalizeDemographicsFields(
async function denormalizeDemographicsFields(
doc: DemographicsDocument,
): Record<string, string | number> {
): Promise<Record<string, string | number>> {
const denormalizedData: Record<string, string | number> = {};
denormalizedData['demographics.ageRange.end'] = doc.ageRange?.end || '';
denormalizedData['demographics.ageRange.start'] = doc.ageRange?.start || '';
const ageRange = await demographicsAgeRange(doc);
denormalizedData['demographics.ageRange.end'] = ageRange?.end || '';
denormalizedData['demographics.ageRange.start'] = ageRange?.start || '';
denormalizedData['demographics.ethnicity'] = doc.ethnicity || '';
denormalizedData['demographics.gender'] = doc.gender || '';
const nationalities =
Expand Down
48 changes: 0 additions & 48 deletions data-serving/data-service/test/model/demographics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import {
demographicsSchema,
} from '../../src/model/demographics';

import { Error } from 'mongoose';
import fullModel from './data/demographics.full.json';
import minimalModel from './data/demographics.minimal.json';
import mongoose from 'mongoose';
Expand All @@ -14,53 +13,6 @@ const Demographics = mongoose.model<DemographicsDocument>(
);

describe('validate', () => {
it('a start age under 0 is invalid', async () => {
return new Demographics({
...fullModel,
...{ ageRange: { start: -0.1 } },
}).validate((e) => {
expect(e).not.toBeNull();
if (e) expect(e.name).toBe(Error.ValidationError.name);
});
});

it('a start age over 120 is invalid', async () => {
return new Demographics({
...fullModel,
...{ ageRange: { start: 121 } },
}).validate((e) => {
expect(e).not.toBeNull();
if (e) expect(e.name).toBe(Error.ValidationError.name);
});
});

it('an end age under 0 is invalid', async () => {
return new Demographics({
...fullModel,
...{ ageRange: { end: -2 } },
}).validate((e) => {
expect(e).not.toBeNull();
if (e) expect(e.name).toBe(Error.ValidationError.name);
});
});

it('a start age without end is valid', async () => {
return new Demographics({
...fullModel,
...{ ageRange: { start: 85 } },
}).validate();
});

it('an end age over 120 is invalid', async () => {
return new Demographics({
...fullModel,
...{ ageRange: { end: 120.1 } },
}).validate((e) => {
expect(e).not.toBeNull();
if (e) expect(e.name).toBe(Error.ValidationError.name);
});
});

it('a minimal demographics document is valid', async () => {
return new Demographics(minimalModel).validate();
});
Expand Down
Loading