diff --git a/api/materialized_views/search/redactedRecordSubset.js b/api/materialized_views/search/redactedRecordSubset.js index df75af623..4ba6e4bc4 100644 --- a/api/materialized_views/search/redactedRecordSubset.js +++ b/api/materialized_views/search/redactedRecordSubset.js @@ -1,239 +1,95 @@ const mongodb = require('../../src/utils/mongodb'); +const defaultLog = require('../../src/utils/logger')('redacted-record-subset'); +const BusinessLogicManager = require('../../src/utils/business-logic-manager'); -const { AUTHORIZED_PUBLISH_AGENCIES } = require('../../src/utils/constants/misc'); -const { SKIP_REDACTION_SCHEMA_NAMES } = require ('../../src/utils/constants/misc'); /** - * Updates the redactedRecord subset. + * Updates or adds the record passed in, in hte redacted record subset * - * @param {*} defaultLog + * @param {*} record */ -async function update(defaultLog) { - // get all records with valid schemaNames - let aggregate = [ - { - $match: { - _schemaName: { - $exists: true - } - } - } - ]; - - const redactCondition = { - $and: [ - { $lt: ['$issuedToAge', 19] }, - { $ne: [{ $arrayElemAt: ['$fullRecord.sourceSystemRef', 0] }, 'nris-epd'] }, - // NRPT-744 ignore ocers-csv records because they have no birthdates. All ocers-csv - // records are pre-redacted - { $ne: [{ $arrayElemAt: ['$fullRecord.sourceSystemRef', 0] }, 'ocers-csv'] } - ] - }; - - const issuedToRedaction = [ - { - $project: { - fullRecord: 1, - issuingAgency: 1, - issuedToAge: { - $cond: { - if: { $ne: [{ $arrayElemAt: ['$fullRecord.issuedTo.dateOfBirth', 0] }, null] }, - then: { - $subtract: [ - { $year: { date: new Date() } }, - { $year: { date: { $arrayElemAt: ['$fullRecord.issuedTo.dateOfBirth', 0] } } } - ] - }, - else: 0 - } - } - } - }, - { - $addFields: { - skipRedact: { - $cond: { - if: { - $in: [{ $arrayElemAt: ['$fullRecord._schemaName', 0] }, SKIP_REDACTION_SCHEMA_NAMES] - }, - then: true, - else: false - } - } - } - }, - { - $addFields: { - 'fullRecord.issuedTo.firstName': { - $cond: { - if: redactCondition, - then: 'Unpublished', - else: { $arrayElemAt: ['$fullRecord.issuedTo.firstName', 0] } - } - }, - 'fullRecord.issuedTo.lastName': { - $cond: { - if: redactCondition, - then: 'Unpublished', - else: { $arrayElemAt: ['$fullRecord.issuedTo.lastName', 0] } - } - }, - 'fullRecord.issuedTo.middleName': { - $cond: { - if: redactCondition, - then: '', - else: { $arrayElemAt: ['$fullRecord.issuedTo.middleName', 0] } - } - }, - 'fullRecord.issuedTo.fullName': { - $cond: { - if: redactCondition, - then: 'Unpublished', - else: { $arrayElemAt: ['$fullRecord.issuedTo.fullName', 0] } - } - }, - 'fullRecord.issuedTo.dateOfBirth': { - $cond: { - if: redactCondition, - then: null, - else: { $arrayElemAt: ['$fullRecord.issuedTo.dateOfBirth', 0] } - } - } - } - }, - // this step will replace issued to with an empty object {} for mines and collections. - // mines and collections don't normally have an issuedTo field, but this should minimize confusion - // TODO: remove the issuedTo field from mines and collections all together - { - $addFields: { - 'fullRecord.issuedTo': { - $cond: { - if: { $eq: ['$skipRedact', true] }, - then: {}, - else: { $arrayElemAt: ['$fullRecord.issuedTo', 0] } - } - } - } - } - ]; +function redactRecord(record) { + let redactedRecord = record.toObject(); + const issuedTo = record.issuedTo; + const issuingAgency = record.issuingAgency; + + if ( BusinessLogicManager.isIssuedToConsideredAnonymous(issuedTo, issuingAgency) ) { + // Remove the issuedTo completely so that it shows up as "Unpublished" on NRCED public. + delete redactedRecord.issuedTo; + + // Make sure that no documents are publicly available either. + redactedRecord.documents = []; + } + + return redactedRecord; +} + + + +/** + * adds the record passed in to the redacted record subset + * + * @param {*} record + */ +async function saveOneRecord(record) { + + const redactedRecord = redactRecord(record); try { + defaultLog.info('Updating redacted_record_subset'); + const db = mongodb.connection.db(process.env.MONGODB_DATABASE || 'nrpti-dev'); - const mainCollection = db.collection('nrpti'); + const redactedCollection = db.collection('redacted_record_subset'); + await redactedCollection.save(redactedRecord); + + defaultLog.info('Done Updating redacted_record_subset'); + } catch (error) { + defaultLog.info('Failed to update redacted_record_subset, error: ' + error); + } +} + +exports.saveOneRecord = saveOneRecord; + + +/** + * Updates the record passed in, in the redacted record subset + * + * @param {*} record + */ +async function updateOneRecord(record) { + + const redactedRecord = redactRecord(record); + + try { defaultLog.info('Updating redacted_record_subset'); - // lookup by id for each object in the match array and populate the fullRecord field - aggregate.push({ - $lookup: { - from: 'nrpti', - localField: '_id', - foreignField: '_id', - as: 'fullRecord' - } - }); - - // redact issued to fields based on age - aggregate = aggregate.concat(issuedToRedaction); - - // replace root with redacted full record - aggregate.push({ - $replaceRoot: { - newRoot: { - $mergeObjects: [{ $arrayElemAt: ['$fullRecord', 0] }, '$$ROOT'] - } - } - }); - - // trim out the 'fullRecord' attribute, we no longer need it - // after re-population - aggregate.push({ - $project: { - fullRecord: 0, - issuedToAge: 0 - } - }); - - const notAuthorizedCondition = { - $and: [ - { - $not: { - $in: ['$issuingAgency', AUTHORIZED_PUBLISH_AGENCIES] - } - }, - { $ne: ['$issuedTo.type', 'Company'] } - ] - }; - - // Remove public from issuedTo.read so the entire issuedTo field is redacted - // when record belongs to unauthorized issuing agency and is not a company - aggregate.push({ - $addFields: { - 'issuedTo.read': { - $cond: { - if: { $and: [{ $eq: ['$skipRedact', false] }, notAuthorizedCondition] }, - then: { - $filter: { - input: '$issuedTo.read', - as: 'role', - cond: { $ne: ['$$role', 'public'] } - } - }, - else: '$issuedTo.read' - } - } - } - }); - - // Remove documents from record when record belongs to unauthorized issuing agency - // and is not a company - aggregate.push({ - $addFields: { - documents: { - $cond: { - if: { $and: [{ $eq: ['$skipRedact', false] }, notAuthorizedCondition] }, - then: [], - else: '$documents' - } - } - } - }); - - // Redaction. We've imported details from - // flavours and documents, and we may need - // to prevent some of these from being returned - // if the user lacks the requisite role(s) - - // for this case, only public users should be using this subset - let roles = ['public']; - - aggregate.push({ - $redact: { - $cond: { - if: { - $cond: { - if: '$read', - then: { - $anyElementTrue: { - $map: { - input: '$read', - as: 'fieldTag', - in: { $setIsSubset: [['$$fieldTag'], roles] } - } - } - }, - else: true - } - }, - then: '$$DESCEND', - else: '$$PRUNE' - } - } - }); - - aggregate.push({ $out: 'redacted_record_subset' }); - - await mainCollection.aggregate(aggregate).next(); + const db = mongodb.connection.db(process.env.MONGODB_DATABASE || 'nrpti-dev'); + const redactedCollection = db.collection('redacted_record_subset'); + await redactedCollection.findOneAndUpdate({ _id: redactedRecord._id }, redactedRecord); + + defaultLog.info('Done Updating redacted_record_subset'); + } catch (error) { + defaultLog.info('Failed to update redacted_record_subset, error: ' + error); + } +} + +exports.updateOneRecord = updateOneRecord; + + + +/** + * Updates the record passed in, in the redacted record subset + * + * @param {*} record + */ +async function deleteOneRecord(record) { + try { + defaultLog.info('Updating redacted_record_subset'); + + const db = mongodb.connection.db(process.env.MONGODB_DATABASE || 'nrpti-dev'); + const redactedCollection = db.collection('redacted_record_subset'); + await redactedCollection.deleteOne({ _id: record._id }); defaultLog.info('Done Updating redacted_record_subset'); } catch (error) { @@ -241,4 +97,4 @@ async function update(defaultLog) { } } -exports.update = update; +exports.deleteOneRecord = deleteOneRecord; diff --git a/api/materialized_views/search/redactedRecordSubset.test.js b/api/materialized_views/search/redactedRecordSubset.test.js index 890151132..35d0af33a 100644 --- a/api/materialized_views/search/redactedRecordSubset.test.js +++ b/api/materialized_views/search/redactedRecordSubset.test.js @@ -1,4 +1,4 @@ -const defaultLog = require('../../src/utils/logger')('redactedRecordSubset.test'); +const mongoose = require('mongoose'); const redactedRecordSubset = require('./redactedRecordSubset'); const { generateIssuedTo } = require('../../test/factories/factory_helper'); @@ -7,6 +7,7 @@ jest.setTimeout(100000); describe('Record Individual issuedTo redaction test', () => { let nrptiCollection = null; let redacted_record_subset = null; + let TestModel = null; beforeAll(async () => { const MongoClient = require('mongodb').MongoClient; @@ -15,6 +16,23 @@ describe('Record Individual issuedTo redaction test', () => { const db = client.db(process.env.MONGODB_DATABASE || 'nrpti-dev'); nrptiCollection = db.collection('nrpti'); redacted_record_subset = db.collection('redacted_record_subset'); + + TestModel = mongoose.model('TestModel', { + issuedTo: { + write: [{ type: String, trim: true, default: 'sysadmin' }], + read: [{ type: String, trim: true, default: 'sysadmin' }], + + type: { type: String, enum: ['Company', 'Individual', 'IndividualCombined'] }, + companyName: { type: String, default: '' }, + firstName: { type: String, default: '' }, + middleName: { type: String, default: '' }, + lastName: { type: String, default: '' }, + fullName: { type: String, default: '' }, + dateOfBirth: { type: Date, default: null } + }, + _schemaName: { type: String, default: '' }, + issuingAgency: { type: String, default: '' } + }); }); beforeEach(async () => { @@ -23,14 +41,14 @@ describe('Record Individual issuedTo redaction test', () => { }); test('Authorized issuing agency and person over 19 are not redacted', async () => { - const testRecord = { - issuedTo: generateIssuedTo( false, true, false ), - _schemaName: 'Schema', - issuingAgency: 'BC Parks' - }; + const testRecord = new TestModel({ + issuedTo: generateIssuedTo( false, true, false ), + _schemaName: 'Schema', + issuingAgency: 'BC Parks' + }); await nrptiCollection.insertOne(testRecord); - await redactedRecordSubset.update(defaultLog); + await redactedRecordSubset.saveOneRecord(testRecord); const redacted = await redacted_record_subset.findOne(); expect(redacted.issuedTo.firstName).toEqual(testRecord.issuedTo.firstName); @@ -41,32 +59,30 @@ describe('Record Individual issuedTo redaction test', () => { }); test('Authorized issuing agency and person under 19 are redacted', async () => { - const testRecord = { + const testRecord = new TestModel({ issuedTo: generateIssuedTo( true, false, false ), _schemaName: 'Schema', issuingAgency: 'BC Parks' - }; + }); await nrptiCollection.insertOne(testRecord); - await redactedRecordSubset.update(defaultLog); + await redactedRecordSubset.saveOneRecord(testRecord); const redacted = await redacted_record_subset.findOne(); - expect(redacted.issuedTo.firstName).toEqual('Unpublished'); - expect(redacted.issuedTo.middleName).toEqual(''); - expect(redacted.issuedTo.lastName).toEqual('Unpublished'); - expect(redacted.issuedTo.fullName).toEqual('Unpublished'); - expect(redacted.issuedTo.dateOfBirth).toEqual(null); + // Expect undefined because entire issuedTo field is redacted + expect(redacted.issuedTo).toEqual(undefined); + expect(redacted.documents).toEqual([]); }); test('Unauthorized issuing agency and person over 19 are redacted', async () => { - const testRecord = { + const testRecord = new TestModel({ issuedTo: generateIssuedTo( false, true, false ), _schemaName: 'Schema', issuingAgency: 'Unauthorized' - }; + }); await nrptiCollection.insertOne(testRecord); - await redactedRecordSubset.update(defaultLog); + await redactedRecordSubset.saveOneRecord(testRecord); const redacted = await redacted_record_subset.findOne(); // Expect undefined because entire issuedTo field is redacted @@ -75,14 +91,15 @@ describe('Record Individual issuedTo redaction test', () => { }); test('Unauthorized issuing agency and person under 19 are redacted', async () => { - const testRecord = { + const testRecord = new TestModel({ issuedTo: generateIssuedTo( true, false, false ), _schemaName: 'Schema', issuingAgency: 'Unauthorized' - }; + }); + await nrptiCollection.insertOne(testRecord); - await redactedRecordSubset.update(defaultLog); + await redactedRecordSubset.saveOneRecord(testRecord); const redacted = await redacted_record_subset.findOne(); // Expect undefined because entire issuedTo field is redacted @@ -91,14 +108,14 @@ describe('Record Individual issuedTo redaction test', () => { }); test('Unauthorized issuing agency and company are not redacted', async () => { - const testRecord = { + const testRecord = new TestModel({ issuedTo: generateIssuedTo( false, false, true ), _schemaName: 'Schema', issuingAgency: 'Unauthorized' - }; + }); await nrptiCollection.insertOne(testRecord); - await redactedRecordSubset.update(defaultLog); + await redactedRecordSubset.saveOneRecord(testRecord); const redacted = await redacted_record_subset.findOne(); expect(redacted.issuedTo.companyName).toEqual(testRecord.issuedTo.companyName); diff --git a/api/openshift/cron.yaml b/api/openshift/cron.yaml index 7419ed79e..2df674db8 100644 --- a/api/openshift/cron.yaml +++ b/api/openshift/cron.yaml @@ -63,12 +63,6 @@ objects: -d '{"taskType":"updateMaterializedView", "materializedViewSubset":"recordNameSubset"}' \ ${NRPTI_API_URL}/task - echo -e "Updating recordName subset\n" - curl -H "Authorization: Bearer ${TOKEN}" \ - -H "Content-Type: application/json" \ - -d '{"taskType":"updateMaterializedView", "materializedViewSubset":"redactedRecordSubset"}' \ - ${NRPTI_API_URL}/task - echo -e "Updating descriptionSummary subset\n" curl -H "Authorization: Bearer ${TOKEN}" \ -H "Content-Type: application/json" \ diff --git a/api/src/tasks/import-task.js b/api/src/tasks/import-task.js index a46d1700c..f83accf43 100644 --- a/api/src/tasks/import-task.js +++ b/api/src/tasks/import-task.js @@ -11,7 +11,6 @@ const issuedToSubset = require('../../materialized_views/search/issuedToSubset') const locationSubset = require('../../materialized_views/search/locationSubset'); const recordNameSubset = require('../../materialized_views/search/recordNameSubset'); const descriptionSummarySubset = require('../../materialized_views/search/descriptionSummarySubset'); -const redactedRecordSubset = require('../../materialized_views/search/redactedRecordSubset'); const outcomeDescriptionSubset = require('../../materialized_views/search/outcomeDescriptionSubset'); @@ -154,9 +153,6 @@ exports.protectedCreateTask = async function(args, res, next) { case 'descriptionSummary': descriptionSummarySubset.update(defaultLog); break; - case 'redactedRecordSubset': - redactedRecordSubset.update(defaultLog); - break; case 'outcomeDescriptionSubset': outcomeDescriptionSubset.update(defaultLog); break; diff --git a/api/src/utils/model-schema-generator.js b/api/src/utils/model-schema-generator.js index 5679755be..c1aa0956e 100644 --- a/api/src/utils/model-schema-generator.js +++ b/api/src/utils/model-schema-generator.js @@ -1,4 +1,6 @@ let mongoose = require('mongoose'); +const redactedRecordSubset = require('../../materialized_views/search/redactedRecordSubset'); +const Constants = require('./constants/misc'); let genSchema = function(name, definition) { // @@ -86,9 +88,20 @@ let genSchema = function(name, definition) { definition._addedBy = { type: String, default: 'system' }; definition._deletedBy = { type: String, default: 'system' }; - schema.post('save', doc => { - // empty func. Audit moved. - }); + if (!Constants.SKIP_REDACTION_SCHEMA_NAMES.includes(name)) { + + schema.post('save', async record => { + await redactedRecordSubset.saveOneRecord(record); + }); + + schema.post('findOneAndUpdate', async record => { + await redactedRecordSubset.updateOneRecord(record); + }); + + schema.post('findOneAndRemove', async record => { + await redactedRecordSubset.deleteOneRecord(record); + }); + } } } if (indexes && indexes.length) { diff --git a/api/src/utils/put-utils.js b/api/src/utils/put-utils.js index 06690bc30..e20997311 100644 --- a/api/src/utils/put-utils.js +++ b/api/src/utils/put-utils.js @@ -370,23 +370,23 @@ exports.editRecordWithFlavours = async function (args, res, next, incomingObj, e } // Update read elements - await MasterModel.updateOne({ _id: masterId }, - { - $set: { - 'read': masterRec.read - } - }, - { new: true }); + await MasterModel.findOneAndUpdate( + { _id: masterId }, + { + $set: { + 'read': masterRec.read + } + }); // Not all records have an issuedTo if (masterRec.issuedTo) { - await MasterModel.updateOne({ _id: masterId }, - { - $set: { - "issuedTo.read": masterRec.issuedTo.read - } - }, - { new: true }); + await MasterModel.findOneAndUpdate( + { _id: masterId }, + { + $set: { + "issuedTo.read": masterRec.issuedTo.read + } + }); } let savedDocuments = null;