diff --git a/angular/projects/admin-nrpti/src/app/import/import-csv/import-csv.component.ts b/angular/projects/admin-nrpti/src/app/import/import-csv/import-csv.component.ts index b561c6912..74ac44391 100644 --- a/angular/projects/admin-nrpti/src/app/import/import-csv/import-csv.component.ts +++ b/angular/projects/admin-nrpti/src/app/import/import-csv/import-csv.component.ts @@ -19,15 +19,17 @@ export class ImportCSVComponent implements OnInit { { displayName: 'AGRI-CMDB', value: 'cmdb-csv' }, { displayName: 'AGRI-MIS', value: 'mis-csv' }, { displayName: 'COORS', value: 'coors-csv' }, - { displayName: 'NRIS-FLNR', value: 'nro-csv' } + { displayName: 'NRIS-FLNR', value: 'flnr-csv' }, + { displayName: 'NRO-ERA', value: 'era-csv' } ]; public csvTypes: any = { 'coors-csv': ['Administrative Sanction', 'Court Conviction', 'Ticket'], - 'nro-csv': ['Inspection'], + 'flnr-csv': ['Inspection'], 'mis-csv': ['Inspection'], 'cmdb-csv': ['Inspection'], - 'alc-csv': ['Inspection'] + 'alc-csv': ['Inspection'], + 'era-csv': ['Ticket'] }; public dataSourceType = null; diff --git a/angular/projects/admin-nrpti/src/app/utils/constants/csv-constants.ts b/angular/projects/admin-nrpti/src/app/utils/constants/csv-constants.ts index 7aff56a80..2f2912262 100644 --- a/angular/projects/admin-nrpti/src/app/utils/constants/csv-constants.ts +++ b/angular/projects/admin-nrpti/src/app/utils/constants/csv-constants.ts @@ -5,6 +5,13 @@ import { alcInspectionsCsvDateFields } from './alc-csv-constants'; +import { + eraTicketsCsvRequiredHeaders, + eraTicketsCsvRequiredFields, + eraTicketsCsvRequiredFormats, + eraTicketsCsvDateFields +} from './era-csv-constants'; + /** * Required format object to specify fields whose value must have a specific format. * @@ -293,7 +300,7 @@ export class CsvConstants { * @static * @memberof CsvConstants */ - public static readonly nroInspectionCsvRequiredHeaders = [ + public static readonly flnrInspectionCsvRequiredHeaders = [ 'Record ID', 'Date', 'Client / Complainant', @@ -312,7 +319,7 @@ export class CsvConstants { * @static * @memberof CsvConstants */ - public static readonly nroInspectionCsvRequiredFields = ['Record ID']; + public static readonly flnrInspectionCsvRequiredFields = ['Record ID']; /** * Fields for NRIS FLNRO NRO Inspection csv that have a required format. @@ -323,7 +330,7 @@ export class CsvConstants { * @type {IRequiredFormat[]} * @memberof CsvConstants */ - public static readonly nroInspectionCsvRequiredFormats: IRequiredFormat[] = [ + public static readonly flnrInspectionCsvRequiredFormats: IRequiredFormat[] = [ { field: 'Date', type: 'date', format: 'YYYY-MM-DD' } ]; @@ -334,7 +341,7 @@ export class CsvConstants { * @type {IDateField[]} * @memberof CsvConstants */ - public static readonly nroInspectionCsvDateFields: IDateField[] = [{ field: 'Date', format: 'YYYY-MM-DD' }]; + public static readonly flnrInspectionCsvDateFields: IDateField[] = [{ field: 'Date', format: 'YYYY-MM-DD' }]; /** * Expected headers for AGRI MIS Inspection csv. @@ -459,9 +466,9 @@ export class CsvConstants { } } - if (dataSourceType === 'nro-csv') { + if (dataSourceType === 'flnr-csv') { if (recordType === 'Inspection') { - return this.nroInspectionCsvRequiredHeaders; + return this.flnrInspectionCsvRequiredHeaders; } } @@ -483,6 +490,12 @@ export class CsvConstants { } } + if (dataSourceType === 'era-csv') { + if (recordType === 'Ticket') { + return eraTicketsCsvRequiredHeaders; + } + } + return null; } @@ -518,9 +531,9 @@ export class CsvConstants { } } - if (dataSourceType === 'nro-csv') { + if (dataSourceType === 'flnr-csv') { if (recordType === 'Inspection') { - return this.nroInspectionCsvRequiredFields; + return this.flnrInspectionCsvRequiredHeaders; } } @@ -542,6 +555,12 @@ export class CsvConstants { } } + if (dataSourceType === 'era-csv') { + if (recordType === 'Ticket') { + return eraTicketsCsvRequiredFields; + } + } + return null; } @@ -577,9 +596,9 @@ export class CsvConstants { } } - if (dataSourceType === 'nro-csv') { + if (dataSourceType === 'flnr-csv') { if (recordType === 'Inspection') { - return this.nroInspectionCsvRequiredFormats; + return this.flnrInspectionCsvRequiredFormats; } } @@ -601,6 +620,12 @@ export class CsvConstants { } } + if (dataSourceType === 'era-csv') { + if (recordType === 'Ticket') { + return eraTicketsCsvRequiredFormats; + } + } + return null; } @@ -636,9 +661,9 @@ export class CsvConstants { } } - if (dataSourceType === 'nro-csv') { + if (dataSourceType === 'flnr-csv') { if (recordType === 'Inspection') { - return this.nroInspectionCsvDateFields; + return this.flnrInspectionCsvDateFields; } } @@ -660,6 +685,12 @@ export class CsvConstants { } } + if (dataSourceType === 'era-csv') { + if (recordType === 'Ticket') { + return eraTicketsCsvDateFields; + } + } + return null; } } diff --git a/angular/projects/admin-nrpti/src/app/utils/constants/era-csv-constants.ts b/angular/projects/admin-nrpti/src/app/utils/constants/era-csv-constants.ts new file mode 100644 index 000000000..27db8b860 --- /dev/null +++ b/angular/projects/admin-nrpti/src/app/utils/constants/era-csv-constants.ts @@ -0,0 +1,56 @@ +import { IRequiredFormat, IDateField } from './csv-constants'; + +/** + * Expected headers for ALC Inspection csv. + * + * Note: sort order and letter case of headers is not important. + */ +export const eraTicketsCsvRequiredHeaders = [ + 'CASE_CONTRAVENTION_ID', + 'ENFORCEMENT_ACTION_ID', + 'FC_CLIENT_NAME', + 'ACT_DESCRIPTION', + 'REG_DESCRIPTION', + 'SECTION', + 'SUB_SECTION', + 'PARAGRAPH', + 'ARTICLE_DESCRIPTION', + 'FINE_AMOUNT', + 'SERVICE_DATE', + 'REGION', + 'ORG_UNIT_NAME', + 'CLIENT_TYPE_CODE' +]; + +/** + * Required fields for ALC Inspection csv. + * + */ +export const eraTicketsCsvRequiredFields = [ + 'CASE_CONTRAVENTION_ID', + 'ENFORCEMENT_ACTION_ID', + 'FC_CLIENT_NAME', + 'ACT_DESCRIPTION', + 'SECTION', + 'ARTICLE_DESCRIPTION', + 'FINE_AMOUNT', + 'SERVICE_DATE', + 'ORG_UNIT_NAME', + 'CLIENT_TYPE_CODE' +]; + +/** + * Fields for ALC Inspection csv that have a required format. + * + * @type {IRequiredFormat[]} + */ +export const eraTicketsCsvRequiredFormats: IRequiredFormat[] = [ + { field: 'SERVICE_DATE', type: 'date', format: 'MM/DD/YYYY' } +]; + +/** + * Fields for ALC Inspection csv that represent dates. + * + * @type {IDateField[]} + */ +export const eraTicketsCsvDateFields: IDateField[] = [{ field: 'SERVICE_DATE', format: 'MM/DD/YYYY' }]; diff --git a/api/src/controllers/post/ticket.js b/api/src/controllers/post/ticket.js index f3716025d..e216c41d1 100644 --- a/api/src/controllers/post/ticket.js +++ b/api/src/controllers/post/ticket.js @@ -99,6 +99,7 @@ exports.createMaster = function(args, res, next, incomingObj, flavourIds) { incomingObj.collectionId && ObjectId.isValid(incomingObj.collectionId) && (ticket.collectionId = new ObjectId(incomingObj.collectionId)); + incomingObj._sourceRefStringId && (ticket._sourceRefStringId = incomingObj._sourceRefStringId); // set permissions ticket.read = utils.ApplicationAdminRoles; @@ -235,6 +236,7 @@ exports.createLNG = function(args, res, next, incomingObj) { ObjectId.isValid(incomingObj._epicMilestoneId) && (ticketLNG._epicMilestoneId = new ObjectId(incomingObj._epicMilestoneId)); incomingObj._sourceRefCoorsId && (ticketLNG._sourceRefCoorsId = incomingObj._sourceRefCoorsId); + incomingObj._sourceRefStringId && (ticketLNG._sourceRefStringId = incomingObj._sourceRefStringId); // set permissions and meta ticketLNG.read = utils.ApplicationAdminRoles; @@ -377,6 +379,7 @@ exports.createNRCED = function(args, res, next, incomingObj) { ObjectId.isValid(incomingObj._epicMilestoneId) && (ticketNRCED._epicMilestoneId = new ObjectId(incomingObj._epicMilestoneId)); incomingObj._sourceRefCoorsId && (ticketNRCED._sourceRefCoorsId = incomingObj._sourceRefCoorsId); + incomingObj._sourceRefStringId && (ticketNRCED._sourceRefStringId = incomingObj._sourceRefStringId); // set permissions and meta ticketNRCED.read = utils.ApplicationAdminRoles; diff --git a/api/src/importers/cmdb/inspection-utils.js b/api/src/importers/cmdb/inspection-utils.js index bda3b4de4..57b222f01 100644 --- a/api/src/importers/cmdb/inspection-utils.js +++ b/api/src/importers/cmdb/inspection-utils.js @@ -37,7 +37,7 @@ const MiscConstants = require('../../utils/constants/misc'); inspection['_sourceRefAgriCmdbId'] = csvRow['inspection id'] || ''; - inspection['recordType'] = 'Inspection'; // TODO: check about inspection type field on csv? there is not currently a "Higher Risk Inspection" recordType + inspection['recordType'] = 'Inspection'; inspection['dateIssued'] = csvRow['date issued'] || null; inspection['issuingAgency'] = 'Ministry of Agriculture'; diff --git a/api/src/importers/era/base-record-utils.js b/api/src/importers/era/base-record-utils.js new file mode 100644 index 000000000..fc2ea0fc0 --- /dev/null +++ b/api/src/importers/era/base-record-utils.js @@ -0,0 +1,157 @@ +'use strict'; + +const mongoose = require('mongoose'); +const defaultLog = require('../../utils/logger')('alc-csv-base-record-utils'); +const RecordController = require('../../controllers/record-controller'); + +/** + * ALC csv base record type handler that can be used directly, or extended if customizations are needed. + * + * @class BaseRecordUtils + */ +class BaseRecordUtils { + /** + * Creates an instance of BaseRecordUtils. + * + * @param {*} auth_payload user information for auditing + * @param {*} recordType an item from record-type-enum.js -> RECORD_TYPE + * @param {*} csvRow an array containing the values from a single csv row. + * @memberof BaseRecordUtils + */ + constructor(auth_payload, recordType, csvRow) { + if (!recordType) { + throw Error('BaseRecordUtils - required recordType must be non-null.'); + } + + this.auth_payload = auth_payload; + this.recordType = recordType; + this.csvRow = csvRow; + } + + /** + * Transform an alc-csv row into a NRPTI record. + * + * Note: Only transforms common fields found in ALL supported alc-csv types. + * To include other values, extend this class and adjust the object returned by this function as needed. + * + * @param {object} csvRow alc-csv row (required) + * @returns {object} NRPTI record. + * @throws {Error} if record is not provided. + * @memberof BaseRecordUtils + */ + transformRecord(csvRow) { + if (!csvRow) { + throw Error('transformRecord - required csvRow must be non-null.'); + } + + return { + _schemaName: this.recordType._schemaName, + recordType: this.recordType.displayName, + + sourceSystemRef: 'era-csv' + }; + } + + /** + * Searches for an existing master record, and returns it if found. + * + * @param {*} nrptiRecord + * @returns {object} existing NRPTI master record, or null if none found or _sourceRefStringId is null + * @memberof BaseRecordUtils + */ + async findExistingRecord(nrptiRecord) { + if (!nrptiRecord._sourceRefStringId) { + return null; + } + + const masterRecordModel = mongoose.model(this.recordType._schemaName); + + return await masterRecordModel + .findOne({ + _schemaName: this.recordType._schemaName, + _sourceRefStringId: nrptiRecord._sourceRefStringId + }) + .populate('_flavourRecords', '_id _schemaName'); + } + + /** + * Update an existing NRPTI master record and its flavour records (if any). + * + * @param {*} nrptiRecord + * @param {*} existingRecord + * @returns {object} object containing the update master and flavour records (if any) + * @memberof BaseRecordUtils + */ + async updateRecord(nrptiRecord, existingRecord) { + if (!nrptiRecord) { + throw Error('updateRecord - required nrptiRecord must be non-null.'); + } + + if (!existingRecord) { + throw Error('updateRecord - required existingRecord must be non-null.'); + } + + try { + // build update Obj, which needs to include the flavour record ids + const updateObj = { ...nrptiRecord, _id: existingRecord._id }; + + updateObj.updatedBy = (this.auth_payload && this.auth_payload.preferred_username) || ''; + updateObj.dateUpdated = new Date(); + + existingRecord._flavourRecords.forEach(flavourRecord => { + updateObj[flavourRecord._schemaName] = { _id: flavourRecord._id, addRole: 'public' }; + }); + + return await RecordController.processPutRequest( + { swagger: { params: { auth_payload: this.auth_payload } } }, + null, + null, + this.recordType.recordControllerName, + [updateObj] + ); + } catch (error) { + defaultLog.error(`Failed to save ${this.recordType._schemaName} record: ${error.message}`); + } + } + + /** + * Create a new NRPTI master and flavour records. + * + * @async + * @param {object} nrptiRecord NRPTI record (required) + * @returns {object} object containing the newly inserted master and flavour records + * @memberof BaseRecordUtils + */ + async createItem(nrptiRecord) { + if (!nrptiRecord) { + throw Error('createItem - required nrptiRecord must be non-null.'); + } + + try { + // build create Obj, which should include the flavour record details + const createObj = { ...nrptiRecord }; + + createObj.addedBy = (this.auth_payload && this.auth_payload.preferred_username) || ''; + createObj.dateAdded = new Date(); + + // publish to NRCED + if (this.recordType.flavours.nrced) { + createObj[this.recordType.flavours.nrced._schemaName] = { + addRole: 'public' + }; + } + + return await RecordController.processPostRequest( + { swagger: { params: { auth_payload: this.auth_payload } } }, + null, + null, + this.recordType.recordControllerName, + [createObj] + ); + } catch (error) { + defaultLog.error(`Failed to create ${this.recordType._schemaName} record: ${error.message}`); + } + } +} + +module.exports = BaseRecordUtils; diff --git a/api/src/importers/era/base-record-utils.test.js b/api/src/importers/era/base-record-utils.test.js new file mode 100644 index 000000000..bc720dec2 --- /dev/null +++ b/api/src/importers/era/base-record-utils.test.js @@ -0,0 +1,132 @@ +const BaseRecordUtils = require('./base-record-utils'); +const RecordController = require('../../controllers/record-controller'); +const RECORD_TYPE = require('../../utils/constants/record-type-enum'); +const MiscConstants = require('../../utils/constants/misc'); + +describe('BaseRecordUtils', () => { + describe('constructor', () => { + it('throws an error if no recordType provided', () => { + expect(() => { + new BaseRecordUtils(null); + }).toThrow('BaseRecordUtils - required recordType must be non-null.'); + }); + }); + + describe('transformRecord', () => { + it('throws error if no csvRow provided', () => { + const baseRecordUtils = new BaseRecordUtils(null, RECORD_TYPE.Ticket); + expect(() => baseRecordUtils.transformRecord(null)).toThrow( + 'transformRecord - required csvRow must be non-null.' + ); + }); + + it('returns transformed csvRow record', () => { + const baseRecordUtils = new BaseRecordUtils(null, RECORD_TYPE.Ticket); + + const csvRow = {}; + + const expectedResult = { + _schemaName: 'Ticket', + recordType: 'Ticket', + + sourceSystemRef: 'era-csv' + }; + + const result = baseRecordUtils.transformRecord(csvRow); + + expect(result).toEqual(expectedResult); + }); + }); + + describe('updateRecord', () => { + it('throws error when nrptiRecord is not provided', async () => { + const baseRecordUtils = new BaseRecordUtils(null, RECORD_TYPE.Ticket); + await expect(baseRecordUtils.updateRecord(null, {})).rejects.toThrow( + 'updateRecord - required nrptiRecord must be non-null.' + ); + }); + + it('throws error when existingRecord is not provided', async () => { + const baseRecordUtils = new BaseRecordUtils(null, RECORD_TYPE.Ticket); + await expect(baseRecordUtils.updateRecord({}, null)).rejects.toThrow( + 'updateRecord - required existingRecord must be non-null.' + ); + }); + + it('calls `processPutRequest` when all arguments provided', async () => { + const baseRecordUtils = new BaseRecordUtils('authPayload', RECORD_TYPE.Ticket); + + const processPutRequestSpy = jest.spyOn(RecordController, 'processPutRequest').mockImplementation(() => { + return Promise.resolve({ test: 'record' }); + }); + + const nrptiRecord = { newField: 'abc', issuedTo: { type: MiscConstants.IssuedToEntityTypes.Individual } }; + const existingRecord = { _id: 123, _flavourRecords: [{ _id: 321, _schemaName: 'flavourSchema' }] }; + + const result = await baseRecordUtils.updateRecord(nrptiRecord, existingRecord); + + expect(processPutRequestSpy).toHaveBeenCalledWith( + { swagger: { params: { auth_payload: 'authPayload' } } }, + null, + null, + RECORD_TYPE.Ticket.recordControllerName, + [ + { + _id: 123, + newField: 'abc', + updatedBy: '', + dateUpdated: expect.any(Date), + flavourSchema: { + _id: 321, + addRole: 'public' + }, + issuedTo: { type: MiscConstants.IssuedToEntityTypes.Individual } + } + ] + ); + + expect(result).toEqual({ test: 'record' }); + }); + }); + + describe('createItem', () => { + it('throws error when nrptiRecord is not provided', async () => { + const baseRecordUtils = new BaseRecordUtils(null, RECORD_TYPE.Ticket); + await expect(baseRecordUtils.createItem(null)).rejects.toThrow( + 'createItem - required nrptiRecord must be non-null.' + ); + }); + + it('calls `processPostRequest` when all arguments provided', async () => { + const baseRecordUtils = new BaseRecordUtils('authPayload', RECORD_TYPE.Ticket); + + const processPostRequestSpy = jest.spyOn(RecordController, 'processPostRequest').mockImplementation(() => { + return Promise.resolve({ test: 'record' }); + }); + + const nrptiRecord = { newField: 'abc', issuedTo: { type: MiscConstants.IssuedToEntityTypes.Individual } }; + + const result = await baseRecordUtils.createItem(nrptiRecord); + + expect(processPostRequestSpy).toHaveBeenCalledWith( + { swagger: { params: { auth_payload: 'authPayload' } } }, + null, + null, + RECORD_TYPE.Ticket.recordControllerName, + [ + { + newField: 'abc', + addedBy: '', + dateAdded: expect.any(Date), + issuedTo: { type: MiscConstants.IssuedToEntityTypes.Individual }, + TicketNRCED: { + addRole: 'public' + } + } + ] + ); + + expect(result).toEqual({ test: 'record' }); + }); + }); +}); diff --git a/api/src/importers/era/datasource.js b/api/src/importers/era/datasource.js new file mode 100644 index 000000000..e2290682d --- /dev/null +++ b/api/src/importers/era/datasource.js @@ -0,0 +1,152 @@ +const defaultLog = require('../../utils/logger')('era-csv-datasource'); +const RECORD_TYPE = require('../../utils/constants/record-type-enum'); + +class EraCsvDataSource { + /** + * Creates an instance of DataSource. + * + * @param {*} taskAuditRecord audit record hook for this import instance + * @param {*} auth_payload information about the user account that started this update + * @param {*} recordType record type to create from the csv file + * @param {*} csvRows array of csv row objects to import + * @memberof EraCsvDataSource + */ + constructor(taskAuditRecord, auth_payload, recordType, csvRows) { + this.taskAuditRecord = taskAuditRecord; + this.auth_payload = auth_payload; + this.recordType = recordType; + this.csvRows = csvRows; + + // Set initial status + this.status = { itemsProcessed: 0, itemTotal: 0, individualRecordStatus: [] }; + } + + /** + * Run the era csv importer. + * + * @returns final status of importer + * @memberof EraCsvDataSource + */ + async run() { + defaultLog.info('run - import era-csv'); + + this.status.itemTotal = this.csvRows.length; + + await this.taskAuditRecord.updateTaskRecord({ status: 'Running', itemTotal: this.csvRows.length }); + + await this.batchProcessRecords(); + + return this.status; + } + + /** + * Runs processRecord() on each csv row, in batches. + * + * Batch size configured by env variable `CSV_IMPORT_BATCH_SIZE` if it exists, or 100 by default. + * + * @memberof EraCsvDataSource + */ + async batchProcessRecords() { + try { + let batchSize = process.env.CSV_IMPORT_BATCH_SIZE || 100; + + const recordTypeConfig = this.getRecordTypeConfig(); + + if (!recordTypeConfig) { + throw Error('batchProcessRecords - failed to find matching recordTypeConfig.'); + } + + let promises = []; + for (let i = 0; i < this.csvRows.length; i++) { + promises.push(this.processRecord(this.csvRows[i], recordTypeConfig)); + + if (i % batchSize === 0 || i === this.csvRows.length - 1) { + await Promise.all(promises); + promises = []; + } + } + } catch (error) { + this.status.message = 'batchProcessRecords - unexpected error'; + this.status.error = error.message; + + defaultLog.error(`batchProcessRecords - unexpected error: ${error.message}`); + } + } + + /** + * Perform all steps necessary to process and save a single row of the csv file. + * + * @param {*} csvRow object of values for a single row + * @param {*} recordTypeConfig object containing record type specific details + * @memberof EraCsvDataSource + */ + async processRecord(csvRow, recordTypeConfig) { + // set status defaults + let recordStatus = {}; + + try { + if (!csvRow) { + throw Error('processRecord - required csvRow is null.'); + } + + if (!recordTypeConfig) { + throw Error('processRecord - required recordTypeConfig is null.'); + } + + // Get a new instance of the record utils that correspond with the current recordType + const recordTypeUtils = recordTypeConfig.getUtil(this.auth_payload, csvRow); + + // Perform any data transformations necessary to convert the csv row into a NRPTI record + const nrptiRecord = recordTypeUtils.transformRecord(csvRow); + + // Check if this record already exists + const existingRecord = await recordTypeUtils.findExistingRecord(nrptiRecord); + + let savedRecord = null; + if (existingRecord) { + // update existing record + savedRecord = await recordTypeUtils.updateRecord(nrptiRecord, existingRecord); + } else { + // create new record + savedRecord = await recordTypeUtils.createItem(nrptiRecord); + } + + if (savedRecord && savedRecord.length > 0 && savedRecord[0].status === 'success') { + this.status.itemsProcessed++; + + await this.taskAuditRecord.updateTaskRecord({ itemsProcessed: this.status.itemsProcessed }); + } else { + throw Error('processRecord - savedRecord is null.'); + } + } catch (error) { + recordStatus.message = 'processRecord - unexpected error'; + recordStatus.error = error.message; + + // only add individual record status when an error occurs + this.status.individualRecordStatus.push(recordStatus); + + // Do not re-throw error, as a single failure is not cause to stop the other records from processing + defaultLog.error(`processRecord - unexpected error: ${error.message}`); + } + } + + /** + * Supported Era csv record type configs. + * + * @returns {*} object with getUtil method to create a new instance of the record type utils. + * @memberof EraCsvDataSource + */ + getRecordTypeConfig() { + if (this.recordType === 'Ticket') { + return { + getUtil: (auth_payload, csvRow) => { + return new (require('./tickets-utils'))(auth_payload, RECORD_TYPE.Ticket, csvRow); + } + }; + } + + return null; + } +} + +module.exports = EraCsvDataSource; diff --git a/api/src/importers/era/datasource.test.js b/api/src/importers/era/datasource.test.js new file mode 100644 index 000000000..fa19f1907 --- /dev/null +++ b/api/src/importers/era/datasource.test.js @@ -0,0 +1,128 @@ +const AlcCsvDataSource = require('./datasource'); + +describe('AlcCsvDataSource', () => { + describe('constructor', () => { + it('sets taskAuditRecord', () => { + const dataSource = new AlcCsvDataSource('taskAuditRecord', null, null, null); + expect(dataSource.taskAuditRecord).toEqual('taskAuditRecord'); + }); + + it('sets auth_payload', () => { + const dataSource = new AlcCsvDataSource(null, 'authPayload', null, null); + expect(dataSource.auth_payload).toEqual('authPayload'); + }); + + it('sets recordType', () => { + const dataSource = new AlcCsvDataSource(null, null, 'recordType', null); + expect(dataSource.recordType).toEqual('recordType'); + }); + + it('sets recordType', () => { + const dataSource = new AlcCsvDataSource(null, null, null, []); + expect(dataSource.csvRows).toEqual([]); + }); + + it('sets default status fields', () => { + const dataSource = new AlcCsvDataSource(); + expect(dataSource.status).toEqual({ + itemsProcessed: 0, + itemTotal: 0, + individualRecordStatus: [] + }); + }); + }); + + describe('processRecord', () => { + it('sets an error if csvRow is null', async () => { + const dataSource = new AlcCsvDataSource(); + + await dataSource.processRecord(null, 'recordTypeConfig'); + + expect(dataSource.status.individualRecordStatus[0]).toEqual({ + message: 'processRecord - unexpected error', + error: 'processRecord - required csvRow is null.' + }); + }); + + it('sets an error if recordTypeConfig is null', async () => { + const dataSource = new AlcCsvDataSource(); + + await dataSource.processRecord('recordTypeConfig', null); + + expect(dataSource.status.individualRecordStatus[0]).toEqual({ + message: 'processRecord - unexpected error', + error: 'processRecord - required recordTypeConfig is null.' + }); + }); + + it('transforms, saves, and updates the status for the new csvRow', async () => { + const taskAuditRecord = { updateTaskRecord: jest.fn(() => {}) }; + + const dataSource = new AlcCsvDataSource(taskAuditRecord, null, null, null); + + const csvRow = {}; + + const recordTypeConfig = { getUtil: () => recordTypeUtils }; + const recordTypeUtils = { + transformRecord: jest.fn(() => { + return { transformed: true }; + }), + findExistingRecord: jest.fn(() => null), + createItem: jest.fn(() => { + return [{ status: 'success' }]; + }), + updateRecord: jest.fn(() => { + return [{ status: 'failure' }]; + }) + }; + + await dataSource.processRecord(csvRow, recordTypeConfig); + + expect(recordTypeUtils.transformRecord).toHaveBeenCalledWith(csvRow); + + expect(recordTypeUtils.findExistingRecord).toHaveBeenCalledWith({ transformed: true }); + + expect(recordTypeUtils.createItem).toHaveBeenCalledWith({ transformed: true }); + + expect(dataSource.status.itemsProcessed).toEqual(1); + + expect(taskAuditRecord.updateTaskRecord).toHaveBeenCalledWith({ itemsProcessed: 1 }); + }); + + it('transforms, saves, and updates the status for the existing csvRow', async () => { + const taskAuditRecord = { updateTaskRecord: jest.fn(() => {}) }; + + const dataSource = new AlcCsvDataSource(taskAuditRecord, null, null, null); + + const csvRow = {}; + + const recordTypeConfig = { getUtil: () => recordTypeUtils }; + const recordTypeUtils = { + transformRecord: jest.fn(() => { + return { transformed: true }; + }), + findExistingRecord: jest.fn(() => { + return { _id: 123 }; + }), + createItem: jest.fn(() => { + return [{ status: 'failure' }]; + }), + updateRecord: jest.fn(() => { + return [{ status: 'success' }]; + }) + }; + + await dataSource.processRecord(csvRow, recordTypeConfig); + + expect(recordTypeUtils.transformRecord).toHaveBeenCalledWith(csvRow); + + expect(recordTypeUtils.findExistingRecord).toHaveBeenCalledWith({ transformed: true }); + + expect(recordTypeUtils.updateRecord).toHaveBeenCalledWith({ transformed: true }, { _id: 123 }); + + expect(dataSource.status.itemsProcessed).toEqual(1); + + expect(taskAuditRecord.updateTaskRecord).toHaveBeenCalledWith({ itemsProcessed: 1 }); + }); + }); +}); diff --git a/api/src/importers/era/tickets-utils.js b/api/src/importers/era/tickets-utils.js new file mode 100644 index 000000000..0a905022c --- /dev/null +++ b/api/src/importers/era/tickets-utils.js @@ -0,0 +1,99 @@ +const BaseRecordUtils = require('./base-record-utils'); +const CsvUtils = require('./utils/csv-utils'); +const MiscConstants = require('../../utils/constants/misc'); + +/** + * ALC csv tickets record handler. + * + * @class tickets + */ +class tickets extends BaseRecordUtils { + /** + * Creates an instance of tickets. + * + * @param {*} auth_payload user information for auditing + * @param {*} recordType an item from record-type-enum.js -> RECORD_TYPE + * @param {*} csvRow an object containing the values from a single csv row. + * @memberof tickets + */ + constructor(auth_payload, recordType, csvRow) { + super(auth_payload, recordType, csvRow); + } + + /** + * Convert the csv row object into the object expected by the API record post/put controllers. + * + * @returns an ticket object matching the format expected by the API record post/put controllers. + * @memberof tickets + */ + transformRecord(csvRow) { + if (!csvRow) { + throw Error('transformRecord - required csvRow must be non-null.'); + } + + const ticket = { ...super.transformRecord(csvRow) }; + + ticket['_sourceRefStringId'] = ''; + + if (csvRow['case_contravention_id'] && csvRow['enforcement_action_id']) { + ticket['_sourceRefStringId'] = `${csvRow['case_contravention_id']}-${csvRow['enforcement_action_id']}`; + } + + ticket['recordName'] = csvRow['article_description'] || ''; + ticket['issuingAgency'] = 'Natural Resource Officers'; + ticket['author'] = 'Natural Resource Officers'; + ticket['recordType'] = 'Ticket'; + ticket['offence'] = csvRow['article_description'] || ''; + ticket['dateIssued'] = csvRow['service_date'] || null; + + if (csvRow['region'] && csvRow['region'] !== '' ) { + ticket['location'] = csvRow['region']; + } else if (csvRow['org_unit_name'] && csvRow['org_unit_name'] !== '') { + ticket['location'] = csvRow['org_unit_name']; + } else { + ticket['location'] = ''; + } + + ticket['penalties'] = [ + { + type: 'Fined', + penalty: { + type: 'Dollars', + value: (csvRow['fine_amount'] && Number(csvRow['fine_amount'])) || null + }, + description:'Penalty Amount (CAD)' + } + ]; + + ticket['legislation'] = { + act: csvRow['act_description'] || '', + regulation: csvRow['reg_description'] || '', + section: csvRow['section'] || '', + subSection: csvRow['sub_section'] || '', + paragraph: csvRow['paragraph'] || '' + }; + + const entityType = CsvUtils.getEntityType(csvRow); + + if (entityType === MiscConstants.IssuedToEntityTypes.Company) { + ticket['issuedTo'] = { + type: MiscConstants.IssuedToEntityTypes.Company, + companyName: csvRow['fc_client_name'] || '' + }; + } + + if (entityType === MiscConstants.IssuedToEntityTypes.Individual) { + ticket['issuedTo'] = { + type: MiscConstants.IssuedToEntityTypes.Individual, + dateOfBirth: null, + firstName: csvRow['fc_client_name'] || '', + lastName: '', + middleName: '', + }; + } + + return ticket; + } +} + +module.exports = tickets; diff --git a/api/src/importers/era/tickets-utils.test.js b/api/src/importers/era/tickets-utils.test.js new file mode 100644 index 000000000..f95a867ce --- /dev/null +++ b/api/src/importers/era/tickets-utils.test.js @@ -0,0 +1,92 @@ +const Tickets = require('./tickets-utils'); +const RECORD_TYPE = require('../../utils/constants/record-type-enum'); +const MiscConstants = require('../../utils/constants/misc'); + +describe('transformRecord', () => { + const tickets = new Tickets('authPayload', RECORD_TYPE.Ticket, null); + + it('throws an error if null csvRow parameter provided', () => { + expect(() => tickets.transformRecord(null)).toThrow('transformRecord - required csvRow must be non-null.'); + }); + + it('returns basic fields if empty csvRow parameter provided', () => { + const result = tickets.transformRecord({}); + + expect(result).toEqual({ + _schemaName: 'Ticket', + _sourceRefStringId: '', + + recordName: '', + recordType: 'Ticket', + issuingAgency: 'Natural Resource Officers', + author: 'Natural Resource Officers', + location: '', + offence: '', + dateIssued: null, + + penalties: [{ description: 'Penalty Amount (CAD)', penalty: { type: 'Dollars', value: null }, type: 'Fined' }], + legislation: { act: '', paragraph: '', regulation: '', section: '', subSection: '' }, + issuedTo: { dateOfBirth: null, firstName: '', lastName: '', middleName: '', type: 'Individual' }, + + sourceSystemRef: 'era-csv', + }); + }); + + it('transforms csv row fields into NRPTI record fields', () => { + const result = tickets.transformRecord({ + case_contravention_id: '123', + enforcement_action_id: '123', + region: 'somewhere', + org_unit_name: 'abcd', + article_description: 'bad manners', + service_date: '02/24/2021', + fine_amount: '1000000', + act_description: 'very rude act', + reg_description: 'asdfasdf', + section: '123', + sub_section: '1234', + paragraph: '12', + fc_client_name: 'bad guy jim' + }); + + expect(result).toEqual({ + _schemaName: 'Ticket', + _sourceRefStringId: '123-123', + + recordName: 'bad manners', + offence: 'bad manners', + location: 'somewhere', + recordType: 'Ticket', + issuingAgency: 'Natural Resource Officers', + author: 'Natural Resource Officers', + dateIssued: '02/24/2021', + + penalties: [{ + description: 'Penalty Amount (CAD)', + penalty: { + type: 'Dollars', + value: 1000000 + }, + type: 'Fined' + }], + + issuedTo: { + dateOfBirth: null, + firstName: 'bad guy jim', + lastName: '', + middleName: '', + type: MiscConstants.IssuedToEntityTypes.Individual + }, + + legislation: { + act: 'very rude act', + regulation: 'asdfasdf', + section: '123', + subSection: '1234', + paragraph: '12' + }, + + sourceSystemRef: 'era-csv' + }); + }); +}); diff --git a/api/src/importers/era/utils/csv-utils.js b/api/src/importers/era/utils/csv-utils.js new file mode 100644 index 000000000..4002e8798 --- /dev/null +++ b/api/src/importers/era/utils/csv-utils.js @@ -0,0 +1,17 @@ +const MiscConstants = require('../../../utils/constants/misc'); + +/** + * Derives the issued to entity type. + * + * @param {*} csvRow + * @returns {string} the entity type. + */ +exports.getEntityType = function(csvRow) { + if (!csvRow) { + return null; + } + + if (csvRow['client_type_code'] === 'C') return MiscConstants.IssuedToEntityTypes.Company; + + return MiscConstants.IssuedToEntityTypes.Individual; +}; diff --git a/api/src/importers/era/utils/csv-utils.test.js b/api/src/importers/era/utils/csv-utils.test.js new file mode 100644 index 000000000..409c6f96b --- /dev/null +++ b/api/src/importers/era/utils/csv-utils.test.js @@ -0,0 +1,28 @@ +const CsvUtils = require('./csv-utils'); +const MiscConstants = require('../../../utils/constants/misc'); + +describe('getEntityType', () => { + it('returns null if null csvRow paramter provided ', async () => { + const result = await CsvUtils.getEntityType(null); + + expect(result).toBe(null); + }); + + it('returns "Company" if csvRow "client_type_code" is "C"', async () => { + const result = await CsvUtils.getEntityType({ 'client_type_code': 'C' }); + + expect(result).toEqual(MiscConstants.IssuedToEntityTypes.Company); + }); + + it('returns "Individual" if csvRow "client_type_code" is empty', async () => { + const result = await CsvUtils.getEntityType({ 'client_type_code': '' }); + + expect(result).toEqual(MiscConstants.IssuedToEntityTypes.Individual); + }); + + it('returns "Individual" if csvRow "client_type_code"', async () => { + const result = await CsvUtils.getEntityType({ 'client_type_code': null }); + + expect(result).toEqual(MiscConstants.IssuedToEntityTypes.Individual); + }); +}); \ No newline at end of file diff --git a/api/src/importers/nro/base-record-utils.js b/api/src/importers/flnr/base-record-utils.js similarity index 94% rename from api/src/importers/nro/base-record-utils.js rename to api/src/importers/flnr/base-record-utils.js index 47040293f..0bbc266d5 100644 --- a/api/src/importers/nro/base-record-utils.js +++ b/api/src/importers/flnr/base-record-utils.js @@ -1,8 +1,8 @@ 'use strict'; const mongoose = require('mongoose'); -const defaultLog = require('../../utils/logger')('nro-csv-base-record-utils'); -const RecordController = require('./../../controllers/record-controller'); +const defaultLog = require('../../utils/logger')('flnr-csv-base-record-utils'); +const RecordController = require('../../controllers/record-controller'); /** * NRO csv base record type handler that can be used directly, or extended if customizations are needed. @@ -29,12 +29,12 @@ class BaseRecordUtils { } /** - * Transform an nro-csv row into a NRPTI record. + * Transform an flnr-csv row into a NRPTI record. * - * Note: Only transforms common fields found in ALL supported nro-csv types. + * Note: Only transforms common fields found in ALL supported flnr-csv types. * To include other values, extend this class and adjust the object returned by this function as needed. * - * @param {object} csvRow nro-csv row (required) + * @param {object} csvRow flnr-csv row (required) * @returns {object} NRPTI record. * @throws {Error} if record is not provided. * @memberof BaseRecordUtils diff --git a/api/src/importers/nro/base-record-utils.test.js b/api/src/importers/flnr/base-record-utils.test.js similarity index 100% rename from api/src/importers/nro/base-record-utils.test.js rename to api/src/importers/flnr/base-record-utils.test.js diff --git a/api/src/importers/nro/datasource.js b/api/src/importers/flnr/datasource.js similarity index 98% rename from api/src/importers/nro/datasource.js rename to api/src/importers/flnr/datasource.js index 0f7245d31..a842bcc4f 100644 --- a/api/src/importers/nro/datasource.js +++ b/api/src/importers/flnr/datasource.js @@ -1,4 +1,4 @@ -const defaultLog = require('../../utils/logger')('nro-csv-datasource'); +const defaultLog = require('../../utils/logger')('flnr-csv-datasource'); const RECORD_TYPE = require('../../utils/constants/record-type-enum'); class NroCsvDataSource { @@ -19,7 +19,7 @@ class NroCsvDataSource { // Only process records that are 'Complete' if(csvRows) { this.csvRows = csvRows.filter(row => row['report status'] && row['report status'] === 'Complete'); - } + } // Set initial status this.status = { itemsProcessed: 0, itemTotal: 0, individualRecordStatus: [] }; diff --git a/api/src/importers/nro/datasource.test.js b/api/src/importers/flnr/datasource.test.js similarity index 100% rename from api/src/importers/nro/datasource.test.js rename to api/src/importers/flnr/datasource.test.js diff --git a/api/src/importers/nro/inspections-utils.js b/api/src/importers/flnr/inspections-utils.js similarity index 100% rename from api/src/importers/nro/inspections-utils.js rename to api/src/importers/flnr/inspections-utils.js diff --git a/api/src/importers/nro/inspections-utils.test.js b/api/src/importers/flnr/inspections-utils.test.js similarity index 100% rename from api/src/importers/nro/inspections-utils.test.js rename to api/src/importers/flnr/inspections-utils.test.js diff --git a/api/src/importers/nro/utils/csv-utils.js b/api/src/importers/flnr/utils/csv-utils.js similarity index 100% rename from api/src/importers/nro/utils/csv-utils.js rename to api/src/importers/flnr/utils/csv-utils.js diff --git a/api/src/importers/nro/utils/csv-utils.test.js b/api/src/importers/flnr/utils/csv-utils.test.js similarity index 100% rename from api/src/importers/nro/utils/csv-utils.test.js rename to api/src/importers/flnr/utils/csv-utils.test.js diff --git a/api/src/models/lng/ticket-lng.js b/api/src/models/lng/ticket-lng.js index ebea5b48b..d7e04e4e0 100644 --- a/api/src/models/lng/ticket-lng.js +++ b/api/src/models/lng/ticket-lng.js @@ -10,6 +10,7 @@ module.exports = require('../../utils/model-schema-generator')( _sourceRefId: { type: 'ObjectId', default: null, index: true }, _epicMilestoneId: { type: 'ObjectId', default: null, index: true }, _master: { type: 'ObjectId', default: null, index: true }, + _sourceRefStringId: { type: String, default: null, index: true }, read: [{ type: String, trim: true, default: 'sysadmin' }], write: [{ type: String, trim: true, default: 'sysadmin' }], diff --git a/api/src/models/master/ticket.js b/api/src/models/master/ticket.js index 8a075e0ea..65271363c 100644 --- a/api/src/models/master/ticket.js +++ b/api/src/models/master/ticket.js @@ -10,6 +10,7 @@ module.exports = require('../../utils/model-schema-generator')( _sourceRefId: { type: 'ObjectId', default: null, index: true }, _epicMilestoneId: { type: 'ObjectId', default: null, index: true }, _sourceRefCoorsId: { type: String, default: null, index: true }, + _sourceRefStringId: { type: String, default: null, index: true }, mineGuid: { type: String, default: null, index: true }, read: [{ type: String, trim: true, default: 'sysadmin' }], diff --git a/api/src/models/nrced/ticket-nrced.js b/api/src/models/nrced/ticket-nrced.js index 8e1c2db45..d292939ca 100644 --- a/api/src/models/nrced/ticket-nrced.js +++ b/api/src/models/nrced/ticket-nrced.js @@ -10,6 +10,7 @@ module.exports = require('../../utils/model-schema-generator')( _sourceRefId: { type: 'ObjectId', default: null, index: true }, _epicMilestoneId: { type: 'ObjectId', default: null, index: true }, _master: { type: 'ObjectId', default: null, index: true }, + _sourceRefStringId: { type: String, default: null, index: true }, read: [{ type: String, trim: true, default: 'sysadmin' }], write: [{ type: String, trim: true, default: 'sysadmin' }], diff --git a/api/src/tasks/import-task.js b/api/src/tasks/import-task.js index 8896f8d1e..3d1d5eebe 100644 --- a/api/src/tasks/import-task.js +++ b/api/src/tasks/import-task.js @@ -229,10 +229,10 @@ function getDataSourceConfig(dataSourceType) { }; } - if (dataSourceType === 'nro-csv') { + if (dataSourceType === 'flnr-csv') { return { - dataSourceLabel: 'nro-csv', - dataSourceClass: require('../importers/nro/datasource') + dataSourceLabel: 'flnr-csv', + dataSourceClass: require('../importers/flnr/datasource') }; } @@ -257,6 +257,13 @@ function getDataSourceConfig(dataSourceType) { } } + if (dataSourceType === 'era-csv') { + return { + dataSourceLabel: 'era-csv', + dataSourceClass: require('../importers/era/datasource') + } + } + // dataSourceType will match the name of a directory for the given // integration in /src/integrations// return {