Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid blocking other requests while validating large output #277

Closed
wants to merge 1 commit into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 24 additions & 16 deletions lib/responseHelper.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,13 @@ responseHelper._enforceSchemaOnArray = (items, schema, callback) => {
if (!(Array.isArray(items))) {
items = [ items ]
}
async.map(items, (item, done) => responseHelper._enforceSchemaOnObject(item, schema, done), (err, results) => {
// the validation process can take a lengthy amount of time when processing large responses.
// to prevent other requests from being blocked by this syncronous process, we break it up using
// setImmediate (see _enforceSchemaOnObject below), and force this map to run in series instead of in parallel.
// It really will not make much of a difference to individual requests, but
// splitting it up this way makes a HUGE difference if a small request arrives on the server while a large request's
// output is still being validated.
async.mapSeries(items, (item, done) => responseHelper._enforceSchemaOnObject(item, schema, done), (err, results) => {
if (err) return callback(err)

results = results.filter(result => !!result)
Expand All @@ -30,24 +36,26 @@ responseHelper._enforceSchemaOnArray = (items, schema, callback) => {
}

responseHelper._enforceSchemaOnObject = (item, schema, callback) => {
debug.validationOutput(JSON.stringify(item))
Joi.validate(item, schema, (err, sanitisedItem) => {
if (err) {
debug.validationError(err.message, JSON.stringify(item))
const res = {
status: '500',
code: 'EINVALIDITEM',
title: 'Item in response does not validate',
detail: {
item: item,
error: err.message
setImmediate(() => {
debug.validationOutput(JSON.stringify(item))
Joi.validate(item, schema, (err, sanitisedItem) => {
if (err) {
debug.validationError(err.message, JSON.stringify(item))
const res = {
status: '500',
code: 'EINVALIDITEM',
title: 'Item in response does not validate',
detail: {
item: item,
error: err.message
}
}
return callback(res)
}
return callback(res)
}

const dataItem = responseHelper._generateDataItem(sanitisedItem, schema)
return callback(null, dataItem)
const dataItem = responseHelper._generateDataItem(sanitisedItem, schema)
return callback(null, dataItem)
})
})
}

Expand Down