NRPT-295: Added ERA csv importer (#791)

* NRPT-295: Added ERA csv  importer

* fixed unit tests

* cleanup

* linting bug

* linting bug

* added record name, and required fields

* swapped client name from fullName to firstName

angular/projects/admin-nrpti/src/app/import/import-csv/import-csv.component.ts

@@ -19,15 +19,17 @@ export class ImportCSVComponent implements OnInit {
     { displayName: 'AGRI-CMDB', value: 'cmdb-csv' },
     { displayName: 'AGRI-MIS', value: 'mis-csv' },
     { displayName: 'COORS', value: 'coors-csv' },
-    { displayName: 'NRIS-FLNR', value: 'nro-csv' }
+    { displayName: 'NRIS-FLNR', value: 'flnr-csv' },
+    { displayName: 'NRO-ERA', value: 'era-csv' }
   ];
 
   public csvTypes: any = {
     'coors-csv': ['Administrative Sanction', 'Court Conviction', 'Ticket'],
-    'nro-csv': ['Inspection'],
+    'flnr-csv': ['Inspection'],
     'mis-csv': ['Inspection'],
     'cmdb-csv': ['Inspection'],
-    'alc-csv': ['Inspection']
+    'alc-csv': ['Inspection'],
+    'era-csv': ['Ticket']
   };
 
   public dataSourceType = null;

angular/projects/admin-nrpti/src/app/utils/constants/csv-constants.ts

@@ -5,6 +5,13 @@ import {
   alcInspectionsCsvDateFields
 } from './alc-csv-constants';
 
+import {
+  eraTicketsCsvRequiredHeaders,
+  eraTicketsCsvRequiredFields,
+  eraTicketsCsvRequiredFormats,
+  eraTicketsCsvDateFields
+} from './era-csv-constants';
+
 /**
  * Required format object to specify fields whose value must have a specific format.
  *
@@ -293,7 +300,7 @@ export class CsvConstants {
    * @static
    * @memberof CsvConstants
    */
-  public static readonly nroInspectionCsvRequiredHeaders = [
+  public static readonly flnrInspectionCsvRequiredHeaders = [
     'Record ID',
     'Date',
     'Client / Complainant',
@@ -312,7 +319,7 @@ export class CsvConstants {
    * @static
    * @memberof CsvConstants
    */
-  public static readonly nroInspectionCsvRequiredFields = ['Record ID'];
+  public static readonly flnrInspectionCsvRequiredFields = ['Record ID'];
 
   /**
    * Fields for NRIS FLNRO NRO Inspection csv that have a required format.
@@ -323,7 +330,7 @@ export class CsvConstants {
    * @type {IRequiredFormat[]}
    * @memberof CsvConstants
    */
-  public static readonly nroInspectionCsvRequiredFormats: IRequiredFormat[] = [
+  public static readonly flnrInspectionCsvRequiredFormats: IRequiredFormat[] = [
     { field: 'Date', type: 'date', format: 'YYYY-MM-DD' }
   ];
 
@@ -334,7 +341,7 @@ export class CsvConstants {
    * @type {IDateField[]}
    * @memberof CsvConstants
    */
-  public static readonly nroInspectionCsvDateFields: IDateField[] = [{ field: 'Date', format: 'YYYY-MM-DD' }];
+  public static readonly flnrInspectionCsvDateFields: IDateField[] = [{ field: 'Date', format: 'YYYY-MM-DD' }];
 
   /**
    * Expected headers for AGRI MIS Inspection csv.
@@ -459,9 +466,9 @@ export class CsvConstants {
       }
     }
 
-    if (dataSourceType === 'nro-csv') {
+    if (dataSourceType === 'flnr-csv') {
       if (recordType === 'Inspection') {
-        return this.nroInspectionCsvRequiredHeaders;
+        return this.flnrInspectionCsvRequiredHeaders;
       }
     }
 
@@ -483,6 +490,12 @@ export class CsvConstants {
       }
     }
 
+    if (dataSourceType === 'era-csv') {
+      if (recordType === 'Ticket') {
+        return eraTicketsCsvRequiredHeaders;
+      }
+    }
+
     return null;
   }
 
@@ -518,9 +531,9 @@ export class CsvConstants {
       }
     }
 
-    if (dataSourceType === 'nro-csv') {
+    if (dataSourceType === 'flnr-csv') {
       if (recordType === 'Inspection') {
-        return this.nroInspectionCsvRequiredFields;
+        return this.flnrInspectionCsvRequiredHeaders;
       }
     }
 
@@ -542,6 +555,12 @@ export class CsvConstants {
       }
     }
 
+    if (dataSourceType === 'era-csv') {
+      if (recordType === 'Ticket') {
+        return eraTicketsCsvRequiredFields;
+      }
+    }
+
     return null;
   }
 
@@ -577,9 +596,9 @@ export class CsvConstants {
       }
     }
 
-    if (dataSourceType === 'nro-csv') {
+    if (dataSourceType === 'flnr-csv') {
       if (recordType === 'Inspection') {
-        return this.nroInspectionCsvRequiredFormats;
+        return this.flnrInspectionCsvRequiredFormats;
       }
     }
 
@@ -601,6 +620,12 @@ export class CsvConstants {
       }
     }
 
+    if (dataSourceType === 'era-csv') {
+      if (recordType === 'Ticket') {
+        return eraTicketsCsvRequiredFormats;
+      }
+    }
+
     return null;
   }
 
@@ -636,9 +661,9 @@ export class CsvConstants {
       }
     }
 
-    if (dataSourceType === 'nro-csv') {
+    if (dataSourceType === 'flnr-csv') {
       if (recordType === 'Inspection') {
-        return this.nroInspectionCsvDateFields;
+        return this.flnrInspectionCsvDateFields;
       }
     }
 
@@ -660,6 +685,12 @@ export class CsvConstants {
       }
     }
 
+    if (dataSourceType === 'era-csv') {
+      if (recordType === 'Ticket') {
+        return eraTicketsCsvDateFields;
+      }
+    }
+
     return null;
   }
 }

angular/projects/admin-nrpti/src/app/utils/constants/era-csv-constants.ts

@@ -0,0 +1,56 @@
+import { IRequiredFormat, IDateField } from './csv-constants';
+
+/**
+ * Expected headers for ALC Inspection csv.
+ *
+ * Note: sort order and letter case of headers is not important.
+ */
+export const eraTicketsCsvRequiredHeaders = [
+  'CASE_CONTRAVENTION_ID',
+  'ENFORCEMENT_ACTION_ID',
+  'FC_CLIENT_NAME',
+  'ACT_DESCRIPTION',
+  'REG_DESCRIPTION',
+  'SECTION',
+  'SUB_SECTION',
+  'PARAGRAPH',
+  'ARTICLE_DESCRIPTION',
+  'FINE_AMOUNT',
+  'SERVICE_DATE',
+  'REGION',
+  'ORG_UNIT_NAME',
+  'CLIENT_TYPE_CODE'
+];
+
+/**
+ * Required fields for ALC Inspection  csv.
+ *
+ */
+export const eraTicketsCsvRequiredFields = [
+  'CASE_CONTRAVENTION_ID',
+  'ENFORCEMENT_ACTION_ID',
+  'FC_CLIENT_NAME',
+  'ACT_DESCRIPTION',
+  'SECTION',
+  'ARTICLE_DESCRIPTION',
+  'FINE_AMOUNT',
+  'SERVICE_DATE',
+  'ORG_UNIT_NAME',
+  'CLIENT_TYPE_CODE'
+];
+
+/**
+ * Fields for ALC Inspection csv that have a required format.
+ *
+ * @type {IRequiredFormat[]}
+ */
+export const eraTicketsCsvRequiredFormats: IRequiredFormat[] = [
+  { field: 'SERVICE_DATE', type: 'date', format: 'MM/DD/YYYY' }
+];
+
+/**
+ * Fields for ALC Inspection csv that represent dates.
+ *
+ * @type {IDateField[]}
+ */
+export const eraTicketsCsvDateFields: IDateField[] = [{ field: 'SERVICE_DATE', format: 'MM/DD/YYYY' }];

api/src/controllers/post/ticket.js

@@ -99,6 +99,7 @@ exports.createMaster = function(args, res, next, incomingObj, flavourIds) {
   incomingObj.collectionId &&
     ObjectId.isValid(incomingObj.collectionId) &&
     (ticket.collectionId = new ObjectId(incomingObj.collectionId));
+  incomingObj._sourceRefStringId && (ticket._sourceRefStringId = incomingObj._sourceRefStringId);
 
   // set permissions
   ticket.read = utils.ApplicationAdminRoles;
@@ -235,6 +236,7 @@ exports.createLNG = function(args, res, next, incomingObj) {
     ObjectId.isValid(incomingObj._epicMilestoneId) &&
     (ticketLNG._epicMilestoneId = new ObjectId(incomingObj._epicMilestoneId));
   incomingObj._sourceRefCoorsId && (ticketLNG._sourceRefCoorsId = incomingObj._sourceRefCoorsId);
+  incomingObj._sourceRefStringId && (ticketLNG._sourceRefStringId = incomingObj._sourceRefStringId);
 
   // set permissions and meta
   ticketLNG.read = utils.ApplicationAdminRoles;
@@ -377,6 +379,7 @@ exports.createNRCED = function(args, res, next, incomingObj) {
     ObjectId.isValid(incomingObj._epicMilestoneId) &&
     (ticketNRCED._epicMilestoneId = new ObjectId(incomingObj._epicMilestoneId));
   incomingObj._sourceRefCoorsId && (ticketNRCED._sourceRefCoorsId = incomingObj._sourceRefCoorsId);
+  incomingObj._sourceRefStringId && (ticketNRCED._sourceRefStringId = incomingObj._sourceRefStringId);
 
   // set permissions and meta
   ticketNRCED.read = utils.ApplicationAdminRoles;

api/src/importers/cmdb/inspection-utils.js

@@ -37,7 +37,7 @@ const MiscConstants = require('../../utils/constants/misc');
 
     inspection['_sourceRefAgriCmdbId'] = csvRow['inspection id'] || '';
 
-    inspection['recordType'] = 'Inspection'; // TODO: check about inspection type field on csv? there is not currently a "Higher Risk Inspection" recordType
+    inspection['recordType'] = 'Inspection';
     inspection['dateIssued'] = csvRow['date issued'] || null;
 
     inspection['issuingAgency'] = 'Ministry of Agriculture';