Merge pull request #58 from unfoldingWord/RJHimprovements

Small improvements to checks
unfoldingWord · Oct 26, 2020 · bbba5f5 · bbba5f5
2 parents c443305 + 2152419
commit bbba5f5
Show file tree

Hide file tree

Showing 9 changed files with 706 additions and 617 deletions.
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "uw-content-validation",
   "description": "Functions for Checking Door43.org Scriptural Content/Resources.",
-  "version": "0.8.16_alpha4",
+  "version": "0.8.16",
   "private": false,
   "homepage": "https://unfoldingword.github.io/uw-content-validation/",
   "repository": {

diff --git a/src/__tests__/__snapshots__/tn-table-row-check.test.js.snap b/src/__tests__/__snapshots__/tn-table-row-check.test.js.snap
@@ -214,6 +214,18 @@ Object {
 exports[`checkTN_TSVDataRow() -  Original Quote tests -  should fail with leading word joiner 1`] = `
 Object {
   "noticeList": Array [
+    Object {
+      "C": "1",
+      "V": "2",
+      "bookID": "GEN",
+      "characterIndex": 0,
+      "extract": "‼וְ‼חֹ֖שֶׁ…",
+      "fieldName": "OrigQuote",
+      "location": " that was supplied",
+      "message": "Unexpected leading word joiner",
+      "priority": 770,
+      "rowID": "d7qw",
+    },
     Object {
       "C": "1",
       "V": "2",
@@ -255,6 +267,18 @@ Object {
 exports[`checkTN_TSVDataRow() -  Original Quote tests -  should fail with leading zero width joiner 1`] = `
 Object {
   "noticeList": Array [
+    Object {
+      "C": "1",
+      "V": "2",
+      "bookID": "GEN",
+      "characterIndex": 0,
+      "extract": "‼וְ⁠חֹ֖שֶׁ…",
+      "fieldName": "OrigQuote",
+      "location": " that was supplied",
+      "message": "Unexpected leading zero-width joiner",
+      "priority": 771,
+      "rowID": "d7qw",
+    },
     Object {
       "C": "1",
       "V": "2",
@@ -390,6 +414,18 @@ Object {
 exports[`checkTN_TSVDataRow() -  Original Quote tests -  should fail with trailing word joiner 1`] = `
 Object {
   "noticeList": Array [
+    Object {
+      "C": "1",
+      "V": "2",
+      "bookID": "GEN",
+      "characterIndex": 0,
+      "extract": "וְ‼חֹ֖שֶׁך…",
+      "fieldName": "OrigQuote",
+      "location": " that was supplied",
+      "message": "Unexpected trailing word joiner",
+      "priority": 772,
+      "rowID": "d7qw",
+    },
     Object {
       "C": "1",
       "V": "2",
@@ -431,6 +467,18 @@ Object {
 exports[`checkTN_TSVDataRow() -  Original Quote tests -  should fail with trailing zero width joiner 1`] = `
 Object {
   "noticeList": Array [
+    Object {
+      "C": "1",
+      "V": "2",
+      "bookID": "GEN",
+      "characterIndex": 0,
+      "extract": "וְ⁠חֹ֖שֶׁך…",
+      "fieldName": "OrigQuote",
+      "location": " that was supplied",
+      "message": "Unexpected trailing zero-width joiner",
+      "priority": 773,
+      "rowID": "d7qw",
+    },
     Object {
       "C": "1",
       "V": "2",
@@ -675,7 +723,8 @@ Object {
       "details": "Found 1 field",
       "location": " that was supplied",
       "message": "Found wrong number of TSV fields (expected 9)",
-      "priority": 861,
+      "priority": 984,
+      "rowID": undefined,
     },
   ],
 }
@@ -691,7 +740,8 @@ Object {
       "details": "Found 1 field",
       "location": " that was supplied",
       "message": "Found wrong number of TSV fields (expected 9)",
-      "priority": 861,
+      "priority": 984,
+      "rowID": undefined,
     },
   ],
 }
@@ -707,7 +757,8 @@ Object {
       "details": "Found 8 fields",
       "location": " that was supplied",
       "message": "Found wrong number of TSV fields (expected 9)",
-      "priority": 861,
+      "priority": 984,
+      "rowID": "w3r5",
     },
   ],
 }

diff --git a/src/__tests__/tn-table-row-check.test.js b/src/__tests__/tn-table-row-check.test.js
@@ -161,14 +161,14 @@ describe('checkTN_TSVDataRow() - ', () => {
     it('should fail with leading word joiner', async() => {
       const chosenLine = "GEN\t1\t2\td7qw\tfigs-imperative\t\u2060וְ⁠חֹ֖שֶׁךְ\t1\tDarkness\tThis is a command. By commanding that light should exist, God made it exist. (See: [[rc://*/ta/man/translate/figs-imperative]])";
       const rawResults = await checkTN_TSVDataRow(languageCode, chosenLine, 'GEN','1','2', 'that was supplied', optionalCheckingOptions);
-      expect(rawResults.noticeList.length).toEqual(3);
+      expect(rawResults.noticeList.length).toEqual(4);
       expect(rawResults).toMatchSnapshot();
     });
 
     it('should fail with trailing word joiner', async() => {
       const chosenLine = "GEN\t1\t2\td7qw\tfigs-imperative\tוְ⁠חֹ֖שֶׁךְ\u2060\t1\tDarkness\tThis is a command. By commanding that light should exist, God made it exist. (See: [[rc://*/ta/man/translate/figs-imperative]])";
       const rawResults = await checkTN_TSVDataRow(languageCode, chosenLine, 'GEN','1','2', 'that was supplied', optionalCheckingOptions);
-      expect(rawResults.noticeList.length).toEqual(3);
+      expect(rawResults.noticeList.length).toEqual(4);
       expect(rawResults).toMatchSnapshot();
     });
 
@@ -189,14 +189,14 @@ describe('checkTN_TSVDataRow() - ', () => {
     it('should fail with leading zero width joiner', async() => {
       const chosenLine = "GEN\t1\t2\td7qw\tfigs-imperative\t\u200dוְ⁠חֹ֖שֶׁךְ\t1\tDarkness\tThis is a command. By commanding that light should exist, God made it exist. (See: [[rc://*/ta/man/translate/figs-imperative]])";
       const rawResults = await checkTN_TSVDataRow(languageCode, chosenLine, 'GEN','1','2', 'that was supplied', optionalCheckingOptions);
-      expect(rawResults.noticeList.length).toEqual(3);
+      expect(rawResults.noticeList.length).toEqual(4);
       expect(rawResults).toMatchSnapshot();
     });
 
     it('should fail with trailing zero width joiner', async() => {
       const chosenLine = "GEN\t1\t2\td7qw\tfigs-imperative\tוְ⁠חֹ֖שֶׁךְ\u200d\t1\tDarkness\tThis is a command. By commanding that light should exist, God made it exist. (See: [[rc://*/ta/man/translate/figs-imperative]])";
       const rawResults = await checkTN_TSVDataRow(languageCode, chosenLine, 'GEN','1','2', 'that was supplied', optionalCheckingOptions);
-      expect(rawResults.noticeList.length).toEqual(3);
+      expect(rawResults.noticeList.length).toEqual(4);
       expect(rawResults).toMatchSnapshot();
     });
 

diff --git a/src/core/annotation-row-check.js b/src/core/annotation-row-check.js
@@ -7,7 +7,7 @@ import { checkTNLinksToOutside } from './tn-links-check';
 import { checkOriginalLanguageQuote } from './quote-check';
 
 
-// const ANNOTATION_TABLE_ROW_VALIDATOR_VERSION_STRING = '0.5.3';
+// const ANNOTATION_TABLE_ROW_VALIDATOR_VERSION_STRING = '0.5.4';
 
 const NUM_EXPECTED_ANNOTATION_TSV_FIELDS = 7; // so expects 6 tabs per line
 const EXPECTED_ANNOTATION_HEADING_LINE = 'Reference\tID\tTags\tSupportReference\tQuote\tOccurrence\tAnnotation';
@@ -479,8 +479,12 @@ export async function checkAnnotationTSVDataRow(languageCode, annotationType, li
             if (annotationType === 'TN')
                 addNoticePartial({ priority: 274, message: "Missing Annotation field", fieldName: 'Annotation', rowID, location: ourRowLocation });
 
-    } else
-        addNoticePartial({ priority: 861, message: `Found wrong number of TSV fields (expected ${NUM_EXPECTED_ANNOTATION_TSV_FIELDS})`, details: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, location: ourRowLocation });
+    } else { // wrong number of fields in the row
+        // Have a go at getting some of the first fields out of the row
+        let rowID = '????';
+        try { rowID = fields[1]; } catch { }
+        addNoticePartial({ priority: 984, message: `Found wrong number of TSV fields (expected ${NUM_EXPECTED_ANNOTATION_TSV_FIELDS})`, details: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, rowID, location: ourRowLocation });
+    }
 
     // console.log(`  checkAnnotationTSVDataRow returning with ${drResult.noticeList.length.toLocaleString()} notice(s).`);
     // console.log("checkAnnotationTSVDataRow result is", JSON.stringify(drResult));

diff --git a/src/core/annotation-table-check.js b/src/core/annotation-table-check.js
@@ -2,7 +2,7 @@ import * as books from './books/books';
 import { checkAnnotationTSVDataRow } from './annotation-row-check';
 
 
-const ANNOTATION_TABLE_VALIDATOR_VERSION_STRING = '0.2.4';
+const ANNOTATION_TABLE_VALIDATOR_VERSION_STRING = '0.2.5';
 
 const NUM_EXPECTED_ANNOTATION_TSV_FIELDS = 7; // so expects 6 tabs per line
 const EXPECTED_TN_HEADING_LINE = 'Reference\tID\tTags\tSupportReference\tQuote\tOccurrence\tAnnotation';
@@ -214,12 +214,18 @@ export async function checkAnnotationRows(languageCode, annotationType, bookID,
 
                 lastC = C; lastV = V;
 
-            } else
+            } else // wrong number of fields in the row
                 // if (n === lines.length - 1) // it's the last line
                 //     console.log(`  Line ${n}: Has ${fields.length} field(s) instead of ${NUM_EXPECTED_TN_FIELDS}: ${EXPECTED_TN_HEADING_LINE.replace(/\t/g, ', ')}`);
                 // else
-                if (n !== lines.length - 1) // it's not the last line
-                    addNoticePartial({ priority: 988, message: `Wrong number of tabbed fields (expected ${NUM_EXPECTED_ANNOTATION_TSV_FIELDS})`, extract: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, lineNumber: n + 1, location: ourLocation });
+                if (n !== lines.length - 1) { // it's not the last line
+                    // Have a go at getting some of the first fields out of the line
+                    let reference = '?:?', C = '?', V = '?', rowID = '????';
+                    try { reference = fields[0]; } catch { }
+                    try { rowID = fields[1]; } catch { }
+                    try { [C, V] = reference.split(':'); } catch { }
+                    addNoticePartial({ priority: 988, message: `Wrong number of tabbed fields (expected ${NUM_EXPECTED_ANNOTATION_TSV_FIELDS})`, extract: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, C, V, rowID, lineNumber: n + 1, location: ourLocation });
+                }
         }
     }
     addSuccessMessage(`Checked all ${(lines.length - 1).toLocaleString()} data line${lines.length - 1 === 1 ? '' : 's'}${ourLocation}.`);

diff --git a/src/core/field-text-check.js b/src/core/field-text-check.js
@@ -106,6 +106,20 @@ export function checkTextField(fieldName, fieldText, allowedLinks, optionalField
             addNoticePartial({ priority: 110, message: `Unexpected leading spaces`, characterIndex: 0, extract, location: ourLocation });
         else
             addNoticePartial({ priority: 109, message: `Unexpected leading space`, characterIndex: 0, extract, location: ourLocation });
+    } else if (fieldText[0] === '\u2060') {
+        const extract = fieldText.substring(0, extractLength).replace(/\u2060/g, '‼') + (fieldText.length > extractLength ? '…' : '');
+        addNoticePartial({ priority: 770, message: `Unexpected leading word joiner`, characterIndex: 0, extract, location: ourLocation });
+    } else if (fieldText[0] === '\u200D') {
+        const extract = fieldText.substring(0, extractLength).replace(/\u200D/g, '‼') + (fieldText.length > extractLength ? '…' : '');
+        addNoticePartial({ priority: 771, message: `Unexpected leading zero-width joiner`, characterIndex: 0, extract, location: ourLocation });
+    }
+
+    if (fieldText[fieldText.length - 1] === '\u2060') {
+        const extract = fieldText.substring(0, extractLength).replace(/\u2060/g, '‼') + (fieldText.length > extractLength ? '…' : '');
+        addNoticePartial({ priority: 772, message: `Unexpected trailing word joiner`, characterIndex: 0, extract, location: ourLocation });
+    } else if (fieldText[fieldText.length - 1] === '\u200D') {
+        const extract = fieldText.substring(0, extractLength).replace(/\u200D/g, '‼') + (fieldText.length > extractLength ? '…' : '');
+        addNoticePartial({ priority: 773, message: `Unexpected trailing zero-width joiner`, characterIndex: 0, extract, location: ourLocation });
     }
 
     const fieldTextLower = fieldText.toLowerCase();

diff --git a/src/core/tn-table-row-check.js b/src/core/tn-table-row-check.js
@@ -7,7 +7,7 @@ import { checkTNLinksToOutside } from './tn-links-check';
 import { checkOriginalLanguageQuote } from './quote-check';
 
 
-// const TN_TABLE_ROW_VALIDATOR_VERSION_STRING = '0.4.7';
+// const TN_TABLE_ROW_VALIDATOR_VERSION_STRING = '0.4.8';
 
 const NUM_EXPECTED_TN_TSV_FIELDS = 9; // so expects 8 tabs per line
 const EXPECTED_TN_HEADING_LINE = 'Book\tChapter\tVerse\tID\tSupportReference\tOrigQuote\tOccurrence\tGLQuote\tOccurrenceNote';
@@ -475,8 +475,12 @@ export async function checkTN_TSVDataRow(languageCode, line, bookID, givenC, giv
         else // TODO: Find out if these fields are really compulsory (and when they're not, e.g., for 'intro') ???
             addNoticePartial({ priority: 274, message: "Missing OccurrenceNote field", fieldName: 'OccurrenceNote', rowID, location: ourRowLocation });
 
-    } else
-        addNoticePartial({ priority: 861, message: `Found wrong number of TSV fields (expected ${NUM_EXPECTED_TN_TSV_FIELDS})`, details: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, location: ourRowLocation });
+    } else { // wrong number of fields in the row
+        // Have a go at getting some of the first fields out of the row
+        let rowID = '????';
+        try { rowID = fields[3]; } catch { }
+        addNoticePartial({ priority: 984, message: `Found wrong number of TSV fields (expected ${NUM_EXPECTED_TN_TSV_FIELDS})`, details: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, rowID, location: ourRowLocation });
+    }
 
     // console.log(`  checkTN_TSVDataRow returning with ${drResult.noticeList.length.toLocaleString()} notice(s).`);
     // console.log("checkTN_TSVDataRow result is", JSON.stringify(drResult));

diff --git a/src/core/tn-table-text-check.js b/src/core/tn-table-text-check.js
@@ -2,7 +2,7 @@ import * as books from './books/books';
 import { checkTN_TSVDataRow } from './tn-table-row-check';
 
 
-const TN_TABLE_TEXT_VALIDATOR_VERSION_STRING = '0.2.3';
+const TN_TABLE_TEXT_VALIDATOR_VERSION_STRING = '0.2.4';
 
 const NUM_EXPECTED_TN_TSV_FIELDS = 9; // so expects 8 tabs per line
 const EXPECTED_TN_HEADING_LINE = 'Book\tChapter\tVerse\tID\tSupportReference\tOrigQuote\tOccurrence\tGLQuote\tOccurrenceNote';
@@ -213,12 +213,18 @@ export async function checkTN_TSVText(languageCode, bookID, filename, tableText,
 
                 lastB = B; lastC = C; lastV = V;
 
-            } else
+            } else // wrong number of fields in the row
                 // if (n === lines.length - 1) // it's the last line
                 //     console.log(`  Line ${n}: Has ${fields.length} field(s) instead of ${NUM_EXPECTED_TN_FIELDS}: ${EXPECTED_TN_HEADING_LINE.replace(/\t/g, ', ')}`);
                 // else
-                if (n !== lines.length - 1) // it's not the last line
-                    addNoticePartial({ priority: 988, message: `Wrong number of tabbed fields (expected ${NUM_EXPECTED_TN_TSV_FIELDS})`, extract: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, lineNumber: n + 1, location: ourLocation });
+                if (n !== lines.length - 1) { // it's not the last line
+                    // Have a go at getting some of the first fields out of the line
+                    let C = '?', V = '?', rowID = '????';
+                    try { C = fields[1]; } catch { }
+                    try { V = fields[2]; } catch { }
+                    try { rowID = fields[3]; } catch { }
+                    addNoticePartial({ priority: 988, message: `Wrong number of tabbed fields (expected ${NUM_EXPECTED_TN_TSV_FIELDS})`, extract: `Found ${fields.length} field${fields.length === 1 ? '' : 's'}`, C, V, rowID, lineNumber: n + 1, location: ourLocation });
+                }
         }
     }
     addSuccessMessage(`Checked all ${(lines.length - 1).toLocaleString()} data line${lines.length - 1 === 1 ? '' : 's'}${ourLocation}.`);