Fix bugs

package.json

@@ -1,7 +1,7 @@
 {
   "name": "uw-content-validation",
   "description": "Functions for Checking Door43.org Scriptural Content/Resources.",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "private": false,
   "homepage": "https://unfoldingword.github.io/uw-content-validation/",
   "repository": {

src/__tests__/__snapshots__/tn-table-row-check.test.js.snap

@@ -151,7 +151,7 @@ Object {
       "C": "1",
       "V": "2",
       "bookID": "GEN",
-      "details": "passage ⸢וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃⸣",
+      "details": "passage ►וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃◄",
       "extract": "וְ⁠חֹ֖שחֹ֖שֶךְ",
       "fieldName": "OrigQuote",
       "location": " that was supplied",
@@ -195,7 +195,7 @@ Object {
       "V": "2",
       "bookID": "GEN",
       "characterIndex": 0,
-      "details": "passage ⸢וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃⸣",
+      "details": "passage ►וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃◄",
       "extract": "(ּ=D1468/H5bc) וְ⁠חֹ֖שֶׁךְ",
       "fieldName": "OrigQuote",
       "location": " that was supplied",
@@ -315,7 +315,7 @@ Object {
       "C": "1",
       "V": "2",
       "bookID": "GEN",
-      "details": "passage ⸢וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃⸣",
+      "details": "passage ►וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃◄",
       "extract": "‌וְ⁠חֹֹ֖֖שֶׁךְ",
       "fieldName": "OrigQuote",
       "location": " that was supplied",
@@ -466,7 +466,7 @@ Object {
       "C": "1",
       "V": "2",
       "bookID": "GEN",
-      "details": "passage ⸢וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃⸣",
+      "details": "passage ►וְ⁠הָ⁠אָ֗רֶץ הָיְתָ֥ה תֹ֨הוּ֙ וָ⁠בֹ֔הוּ וְ⁠חֹ֖שֶׁךְ עַל־פְּנֵ֣י תְה֑וֹם וְ⁠ר֣וּחַ אֱלֹהִ֔ים מְרַחֶ֖פֶת עַל־פְּנֵ֥י הַ⁠מָּֽיִם׃◄",
       "extract": "וְ⁠חֹ֖ש֖שֶׁךְ‌",
       "fieldName": "OrigQuote",
       "location": " that was supplied",

src/core/annotation-table-check.js

@@ -57,7 +57,7 @@ export async function checkAnnotationRows(languageCode, annotationType, bookID,
         console.assert(typeof noticeObject.location === 'string', `TSV addNoticePartial: 'location' parameter should be a string not a '${typeof noticeObject.location}': ${noticeObject.location}`);
 
         if (noticeObject.debugChain) noticeObject.debugChain = `checkAnnotationRows ${noticeObject.debugChain}`;
-        carResult.noticeList.push({ ...noticeObject, bookID, filename });
+        carResult.noticeList.push({ ...noticeObject, bookID, filename, repoCode: annotationType });
     }
 
 

src/core/disabled-notices.js

@@ -8,7 +8,7 @@
 */
 
 
-// const DISABLED_NOTICES_VERSION_STRING = '0.3.0';
+// const DISABLED_NOTICES_VERSION_STRING = '0.3.1';
 
 
 const disabledNotices = [
@@ -69,7 +69,9 @@ const disabledNotices = [
  * @returns true if the givenNotice has a match in the disabledNotices list above
  */
 export function isDisabledNotice(givenNotice) {
+  // NOTE: The function will fail if repoCode is not set in the notices passed to this function
   // console.log(`isDisabledNotice(${JSON.stringify(givenNotice)})…`);
+  // if (givenNotice.repoCode === undefined) console.log(`isDisabledNotice() cannot work without repoCode for ${JSON.stringify(givenNotice)}`);
   for (const disabledNotice of disabledNotices) {
     let matchedAllSpecifiedFields = true;
     for (const propertyName in disabledNotice)
@@ -94,8 +96,11 @@ export function isDisabledNotice(givenNotice) {
  * @returns a new list of notices with disabled ones removed
  */
 export function removeDisabledNotices(givenNoticeList) {
+  // NOTE: The function will fail if repoCode is not set in the notices passed to this function
   const remainingNoticeList = [];
+  let givenRepoCodeNotice = false;
   for (const thisNotice of givenNoticeList) {
+    if (thisNotice.repoCode === undefined && !givenRepoCodeNotice) { console.log(`removeDisabledNotices() cannot work without repoCode for ${JSON.stringify(thisNotice)} in list of ${givenNoticeList.length} notices.`); givenRepoCodeNotice = true; }
     if (!isDisabledNotice(thisNotice))
       remainingNoticeList.push(thisNotice);
     // else console.log(`  Removing disabled ${JSON.stringify(thisNotice)}`);

src/core/orig-quote-check.js

@@ -4,7 +4,7 @@ import { cachedGetFile } from '../core/getApi';
 import { ourParseInt } from './utilities';
 
 
-// const QUOTE_VALIDATOR_VERSION_STRING = '0.7.7';
+// const QUOTE_VALIDATOR_VERSION_STRING = '0.8.0';
 
 
 export async function checkOriginalLanguageQuote(languageCode, fieldName, fieldText, occurrenceString, bookID, C, V, givenLocation, checkingOptions) {
@@ -287,18 +287,27 @@ export async function checkOriginalLanguageQuote(languageCode, fieldName, fieldT
         }
         const numQuoteBits = quoteBits.length;
         if (numQuoteBits >= 2) {
+            let quoteIndex = -1; // These parts have to be in order, i.e., found in the verse one AFTER the other
             for (let bitIndex = 0; bitIndex < numQuoteBits; bitIndex++) {
-                if (verseText.indexOf(quoteBits[bitIndex]) < 0) { // this is what we really want to catch
+                // console.log(`Checking quote part ${bitIndex} '${quoteBits[bitIndex]}' in '${verseText.substring(quoteIndex)}' from '${verseText}'`)
+                if ((quoteIndex = verseText.indexOf(quoteBits[bitIndex], quoteIndex + 1)) < 0) { // this is what we really want to catch
                     // If the quote has multiple parts, create a description of the current part
                     let partDescription;
                     if (numQuoteBits === 1) partDescription = '';
                     else if (bitIndex === 0) partDescription = 'beginning';
                     else if (bitIndex === numQuoteBits - 1) partDescription = 'end';
                     else partDescription = `middle${numQuoteBits > 3 ? bitIndex : ''}`;
-                    // console.log(`721 Unable to find '${fieldText}' ${numQuoteBits === 1? '': `'${quoteBits[bitIndex]}' `}${partDescription? '('+partDescription+') ':''}in '${verseText}'`);
-                    const extract = `${quoteBits[bitIndex]}' ${partDescription ? '(' + partDescription + ')' : ''}`;
-                    addNotice({ priority: 721, message: "Unable to find original language quote in verse text", extract, location: ourLocation });
+                    const extract = `${partDescription ? '(' + partDescription + ' quote portion)' : ''} '${quoteBits[bitIndex]}'`;
+                    if (verseText.indexOf(quoteBits[bitIndex]) >= 0) {
+                        console.assert(bitIndex > 0, "This shouldn't happen for bitIndex of zero!");
+                        // console.log(`914, Unable to find '${fieldText}' ${numQuoteBits === 1 ? '' : `'${quoteBits[bitIndex]}' `}${partDescription ? '(' + partDescription + ') ' : ''}in '${verseText}'`);
+                        addNotice({ priority: 914, message: "Unable to find original language quote portion in the right place in the verse text", details: `passage ►${verseText}◄`, extract, location: ourLocation });
+                    } else {
+                        // console.log(`915, Unable to find '${fieldText}' ${numQuoteBits === 1 ? '' : `'${quoteBits[bitIndex]}' `}${partDescription ? '(' + partDescription + ') ' : ''}in '${verseText}'`);
+                        addNotice({ priority: 915, message: "Unable to find original language quote portion in verse text", details: `passage ►${verseText}◄`, extract, location: ourLocation });
+                    }
                 }
+                // else console.log(`Found quote ${bitIndex} at ${quoteIndex} (num text chars = ${verseText.length})`);
             }
         } else // < 2
             addNotice({ priority: 375, message: "Ellipsis without surrounding snippet", location: ourLocation });
@@ -308,7 +317,7 @@ export async function checkOriginalLanguageQuote(languageCode, fieldName, fieldT
                 // console.log(`checkOriginalLanguageQuote is checking for ${occurrence} occurrences of ${fieldText}`);
                 if (verseText.split(fieldText).length <= occurrence) { // There's not enough of them
                     const extract = fieldText.substring(0, halfLength) + (fieldText.length > 2 * halfLength ? '…' : '') + fieldText.substring(fieldText.length - halfLength, fieldText.length);
-                    addNotice({ priority: 917, message: "Unable to find duplicate original language quote in verse text", details: `occurrence=${occurrenceString}, passage ⸢${verseText}⸣`, extract, location: ourLocation });
+                    addNotice({ priority: 917, message: "Unable to find duplicate original language quote in verse text", details: `occurrence=${occurrenceString}, passage ►${verseText}◄`, extract, location: ourLocation });
                 }
             } else { // We only need to check for one occurrence
                 // Double check that it doesn’t start/stop in the middle of a word
@@ -325,7 +334,7 @@ export async function checkOriginalLanguageQuote(languageCode, fieldName, fieldT
                     // const badCharString = ` by '{badChar}' {unicodedata.name(badChar)}={hex(ord(badChar))}`;
                     // console.log(`Seems '${fieldText}' might not start at the beginning of a word—it’s preceded ${badCharString} in '${verseText}'`);
                     const extract = `(${remainingBits[0].slice(-1)}=D${remainingBits[0].slice(-1).charCodeAt()}/H${remainingBits[0].slice(-1).charCodeAt().toString(16)})` + fieldText.substring(0, extractLength - 3) + (fieldText.length > extractLength - 3 ? '…' : '');
-                    addNotice({ priority: 620, message: "Seems original language quote might not start at the beginning of a word", details: `passage ⸢${verseText}⸣`, characterIndex: 0, extract, location: ourLocation });
+                    addNotice({ priority: 620, message: "Seems original language quote might not start at the beginning of a word", details: `passage ►${verseText}◄`, characterIndex: 0, extract, location: ourLocation });
                 }
                 // Note: There's some Hebrew (RTL) characters at the beginning of the following regex
                 if (fieldText.slice(-1) !== ' ' && remainingBits[1] && remainingBits[1][0].search(/[^׃־A-Za-z\s.,:;?!–)]…/) !== -1) {
@@ -343,7 +352,13 @@ export async function checkOriginalLanguageQuote(languageCode, fieldName, fieldT
             if (noBreakSpaceText) fieldText = fieldText.replace(/\u00A0/g, '⍽');
             // console.log(`722 fieldText='${fieldText}'${extraText}`);
             // console.log(`722 verseText='${verseText}'`);
-            if (fieldText[0] === '\u2060') { // Word joiner
+            if (fieldText[0] === ' ') {
+                const extract = fieldText.substring(0, extractLength) + fieldText.length > extractLength ? '…' : '';
+                addNotice({ priority: 916, message: "Unable to find original language quote in verse text", details: "quote which starts with a space" + (noBreakSpaceText ? ' ' + noBreakSpaceText : ''), extract, location: ourLocation });
+            } else if (fieldText.endsWith(' ')) {
+                const extract = fieldText.length > extractLength ? '…' : '' + fieldText.substring(fieldText.length - extractLength, fieldText.length);
+                addNotice({ priority: 916, message: "Unable to find original language quote in verse text", details: "quote which ends with a space" + (noBreakSpaceText ? ' ' + noBreakSpaceText : ''), extract, location: ourLocation });
+            } else if (fieldText[0] === '\u2060') { // Word joiner
                 const extract = fieldText.substring(0, extractLength) + fieldText.length > extractLength ? '…' : '';
                 addNotice({ priority: 916, message: "Unable to find original language quote in verse text", details: "quote which starts with 'word joiner'" + (noBreakSpaceText ? ' ' + noBreakSpaceText : ''), extract, location: ourLocation });
             } else if (fieldText.endsWith('\u2060')) { // Word joiner
@@ -363,7 +378,7 @@ export async function checkOriginalLanguageQuote(languageCode, fieldName, fieldT
                 addNotice({ priority: 916, message: "Unable to find original language quote in verse text", details: "quote which ends with 'zero-width joiner'" + (noBreakSpaceText ? ' ' + noBreakSpaceText : ''), extract, location: ourLocation });
             } else {
                 const extract = fieldText.substring(0, halfLength) + (fieldText.length > 2 * halfLength ? '…' : '') + fieldText.substring(fieldText.length - halfLength, fieldText.length);
-                addNotice({ priority: 916, message: "Unable to find original language quote in verse text", details: noBreakSpaceText ? noBreakSpaceText : `passage ⸢${verseText}⸣`, extract, location: ourLocation });
+                addNotice({ priority: 916, message: "Unable to find original language quote in verse text", details: noBreakSpaceText ? noBreakSpaceText : `passage ►${verseText}◄`, extract, location: ourLocation });
             }
         }
     }

src/core/tn-table-text-check.js

@@ -4,7 +4,7 @@ import { checkTN_TSVDataRow } from './tn-table-row-check';
 import { removeDisabledNotices } from './disabled-notices';
 
 
-const TN_TABLE_TEXT_VALIDATOR_VERSION_STRING = '0.3.1';
+const TN_TABLE_TEXT_VALIDATOR_VERSION_STRING = '0.3.2';
 
 const NUM_EXPECTED_TN_TSV_FIELDS = 9; // so expects 8 tabs per line
 const EXPECTED_TN_HEADING_LINE = 'Book\tChapter\tVerse\tID\tSupportReference\tOrigQuote\tOccurrence\tGLQuote\tOccurrenceNote';
@@ -68,6 +68,7 @@ export async function checkTN_TSVText(languageCode, bookID, filename, tableText,
         if (noticeObject.debugChain) noticeObject.debugChain = `checkTN_TSVText ${noticeObject.debugChain}`;
         // NOTE: We only add the repoCode here because this function is called directly by tC Create
         //          and notice disabling currently depends on knowing the repoCode
+        if (noticeObject.repoCode) console.log(`checkTN_TSVText.addNoticePartial already had repoCode=${noticeObject.repoCode} (will be lost)`);
         ttResult.noticeList.push({ ...noticeObject, bookID, filename, repoCode: 'TN' });
     }