Bug fixes

README.md

@@ -102,7 +102,7 @@ There is provision for checking to be altered and/or sped-up when the calling ap
 - `taRepoLanguageCode`, and `taRepoSectionName`: can be used to specify how the `SupportReference` field is checked in TA—defaults are 'en' and 'translate'
 - `twRepoUsername`, `twRepoBranchName`: these two fields can be used to specify the username/organisation and/or the branch/tag name for fetching the TW files for checking
 - `extractLength`: an integer which defines how long excerpts of lines containing errors should be—the default is 15 characters—the package attempts to place the error in the middle of the extract
-- `cutoffPriorityLevel`: an integer which can define notices to not be detected—defaults to 0 so none are dropped. Note that this will also affect the `suggestion` response.
+- `cutoffPriorityLevel`: an integer which can define notices to not be detected—defaults to 0 so none are dropped. Note that this will also affect the `suggestion` response. (Only partially implemented at present, so drops some but not all low priority notices.)
 - `suppressNoticeDisablingFlag`: Defaults to `false`, i.e., to removing (thus suppressing) notices for warnings which are expected in certain files and hence we don’t want them displayed. Note that this is always set to `true` for the demos (because they suppress these notices later—see the `showDisabledNoticesFlag` below).
 
     Currently this supressing is only done in the (exported) `checkTN_TSVText` and `checkAnnotationRows` functions which we know to be called by [tC Create](https://github.com/unfoldingWord/tc-create-app) as well as `checkManifestText`, `checkMarkdownText`, `checkPlainText`, `checkTN_TSVText`, `checkUSFMText`, and `checkYAMLText` called by the [Content Validation App](https://github.com/unfoldingWord-box3/content-validation-app).
@@ -155,7 +155,7 @@ Known bugs:
 Known check deficiencies:
 
 1. Markdown image format `![xx](yy)` is not yet fully checked
-1. Filenames in manifests are not fully checked
+1. Manifests are not checked against all files, i.e., to find files potentially missing from the manifest
 1. Naked HTTP links are not yet checked properly
 1. ULT/UST quotes in TranslationAcademy are not yet checked
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "uw-content-validation",
   "description": "Functions for Checking Door43.org Scriptural Content/Resources.",
-  "version": "0.9.3",
+  "version": "1.0.0",
   "private": false,
   "homepage": "https://unfoldingword.github.io/uw-content-validation/",
   "repository": {

src/__tests__/__snapshots__/book-package-check.test.js.snap

@@ -204,18 +204,6 @@ Object {
       "repoName": "hbo_uhb",
       "username": "unfoldingWord",
     },
-    Object {
-      "bookID": "RUT",
-      "details": "username=unfoldingWord error=Could not find src/__tests__/fixtures/unfoldingWord/en_twl/RUT_twl.tsv",
-      "extra": "TWL",
-      "filename": "RUT_twl.tsv",
-      "location": " in TWL in en RUT book package from unfoldingWord master branch",
-      "message": "Unable to load",
-      "priority": 996,
-      "repoCode": "TWL",
-      "repoName": "en_twl",
-      "username": "unfoldingWord",
-    },
     Object {
       "C": "2",
       "V": "1",
@@ -7172,18 +7160,6 @@ Object {
     "en_tq",
   ],
   "noticeList": Array [
-    Object {
-      "bookID": "TIT",
-      "details": "username=unfoldingWord error=Could not find src/__tests__/fixtures/unfoldingWord/en_twl/TIT_twl.tsv",
-      "extra": "TWL",
-      "filename": "TIT_twl.tsv",
-      "location": " in TWL in en TIT book package from unfoldingWord master branch",
-      "message": "Unable to load",
-      "priority": 996,
-      "repoCode": "TWL",
-      "repoName": "en_twl",
-      "username": "unfoldingWord",
-    },
     Object {
       "C": "1",
       "V": "1",

src/core/manifest-text-check.js

@@ -6,7 +6,7 @@ import Ajv from 'ajv';
 import { removeDisabledNotices } from './disabled-notices';
 
 
-const MANIFEST_VALIDATOR_VERSION_STRING = '0.3.7';
+const MANIFEST_VALIDATOR_VERSION_STRING = '0.4.0';
 
 // Pasted in 2020-10-02 from https://raw.githubusercontent.com/unfoldingWord/dcs/master/options/schema/rc.schema.json
 const MANIFEST_SCHEMA = {

src/demos/book-package-check/checkBookPackage.js

@@ -5,7 +5,7 @@ import { checkFileContents } from '../file-check/checkFileContents';
 import { checkRepo } from '../repo-check/checkRepo';
 
 
-// const BP_VALIDATOR_VERSION_STRING = '0.5.2';
+// const BP_VALIDATOR_VERSION_STRING = '0.5.3';
 
 const MANIFEST_FILENAME = 'manifest.yaml';
 
@@ -46,7 +46,7 @@ export async function checkBookPackage(username, languageCode, bookID, setResult
   let checkedFileCount = 0, checkedFilenames = [], checkedFilenameExtensions = new Set(), totalCheckedSize = 0, checkedRepoNames = new Set();
   let checkBookPackageResult = { successList: [], noticeList: [] };
 
-  let dataSet = checkingOptions.dataSet; // Can be 'DEFAULT', 'OLD' (Markdown, etc.), 'NEW' (TSV only), or 'BOTH'
+  let dataSet = checkingOptions?.dataSet; // Can be 'DEFAULT', 'OLD' (Markdown, etc.), 'NEW' (TSV only), or 'BOTH'
   if (!dataSet) dataSet = 'DEFAULT';
 
   const newCheckingOptions = checkingOptions ? { ...checkingOptions } : {}; // clone before modify
@@ -112,7 +112,7 @@ export async function checkBookPackage(username, languageCode, bookID, setResult
     console.assert(typeof fileLocation === 'string', `cBP ourCheckBPFileContents: 'fileLocation' parameter should be a string not a '${typeof fileLocation}'`);
     console.assert(checkingOptions !== undefined, "cBP ourCheckBPFileContents: 'checkingOptions' parameter should be defined");
 
-    const cfcResultObject = await checkFileContents(languageCode, repoCode, cfFilename, fileContent, fileLocation, checkingOptions);
+    const cfcResultObject = await checkFileContents(username, languageCode, repoCode, originalBranch, cfFilename, fileContent, fileLocation, checkingOptions);
     // console.log("checkFileContents() returned", cfResultObject.successList.length, "success message(s) and", cfResultObject.noticeList.length, "notice(s)");
     // for (const successEntry of cfResultObject.successList) console.log("  ourCheckBPFileContents:", successEntry);
     // console.log("cfcResultObject", JSON.stringify(cfcResultObject));
@@ -288,7 +288,7 @@ export async function checkBookPackage(username, languageCode, bookID, setResult
     // So now we want to work through checking this one specified Bible book in various repos
     const origLangRepoCode = whichTestament === 'old' ? 'UHB' : 'UGNT';
     if (dataSet === 'DEFAULT')
-      repoCodeList = languageCode === 'en' ? [origLangRepoCode, 'TWL', 'LT', 'ST', 'TN', 'TQ'] : [origLangRepoCode, 'TWL', 'LT', 'ST', 'TN', 'TQ'];
+      repoCodeList = languageCode === 'en' ? [origLangRepoCode, 'LT', 'ST', 'TN', 'TQ'] : [origLangRepoCode, 'LT', 'ST', 'TN', 'TQ'];
     else if (dataSet === 'OLD')
       repoCodeList = languageCode === 'en' ? [origLangRepoCode, 'LT', 'ST', 'TN', 'TQ'] : [origLangRepoCode, 'LT', 'ST', 'TN', 'TQ'];
     else if (dataSet === 'NEW')
@@ -509,7 +509,7 @@ async function checkTQMarkdownBook(username, languageCode, repoName, branch, boo
     console.assert(typeof fileLocation === 'string', `cTQ ourCheckTQFileContents: 'fileLocation' parameter should be a string not a '${typeof fileLocation}'`);
     console.assert(checkingOptions !== undefined, "cTQ ourCheckTQFileContents: 'checkingOptions' parameter should be defined");
 
-    const cfResultObject = await checkFileContents(languageCode, repoCode, cfFilename, fileContent, fileLocation, checkingOptions);
+    const cfResultObject = await checkFileContents(username, languageCode, repoCode, branch, cfFilename, fileContent, fileLocation, checkingOptions);
     // console.log("checkFileContents() returned", cfResultObject.successList.length, "success message(s) and", cfResultObject.noticeList.length, "notice(s)");
     // for (const successEntry of cfResultObject.successList) console.log("  ourCheckTQFileContents:", successEntry);
 

src/demos/file-check/FileCheck.js

@@ -75,7 +75,7 @@ function FileCheck(props) {
         else if (repoName.endsWith('lt')) repoCodeGuess = 'LT'
         else if (repoName.endsWith('st')) repoCodeGuess = 'ST'
 
-        rawCFResults = await checkFileContents(languageCode, repoCodeGuess, filename, fileContent, givenLocation, checkingOptions);
+        rawCFResults = await checkFileContents(username, languageCode, repoCodeGuess, branch, filename, fileContent, givenLocation, checkingOptions);
 
         // Because we know here that we're only checking one file, we don’t need the filename field in the notices
         // WRONG: We want the filename so that the lineNumber can be made into a live link
@@ -126,7 +126,7 @@ function FileCheck(props) {
             {processedResults.numIgnoredNotices ? `${processedResults.numIgnoredNotices.toLocaleString()} ignored notice(s)` : ""}
             {processedResults.numIgnoredNotices && processedResults.numDisabledNotices ? ' and ' : ''}
             {processedResults.numDisabledNotices ? `${processedResults.numDisabledNotices.toLocaleString()} expected/disabled notice(s)` : ""}
-            {processedResults.numIgnoredNotices || processedResults.numDisabledNotices ? ')' : ''}.{rawCFResults.checkedOptions.cutoffPriorityLevel? ` Priority level ${rawCFResults.checkedOptions.cutoffPriorityLevel} or lower were not included.`:''}</p>
+            {processedResults.numIgnoredNotices || processedResults.numDisabledNotices ? ')' : ''}.{rawCFResults.checkedOptions.cutoffPriorityLevel ? ` Priority level ${rawCFResults.checkedOptions.cutoffPriorityLevel} or lower were not included.` : ''}</p>
           {/* <RenderRawResults results={rawCFResults} /> */}
         </div>);
       }

src/demos/file-check/checkFileContents.js

@@ -2,31 +2,38 @@ import * as books from '../../core/books/books';
 import { formRepoName, checkUSFMText, checkMarkdownFileContents, checkPlainText, checkYAMLText, checkManifestText, checkTN_TSVText, checkAnnotationRows } from '../../core';
 
 
-// const CHECK_FILE_CONTENTS_VERSION_STRING = '0.2.3';
+// const CHECK_FILE_CONTENTS_VERSION_STRING = '0.3.1';
 
 
 /**
  *
- * @param {string} languageCode
+ * @param {string} username for Door43.org
+ * @param {string} languageCode, e.g., 'en'
+ * @param {string} repoCode, e.g., 'TN'
+ * @param {string} branch, e.g., 'master'
  * @param {string} filename
  * @param {string} fileContent
  * @param {string} givenLocation
  * @param {Object} checkingOptions
  */
-export async function checkFileContents(languageCode, repoCode, filename, fileContent, givenLocation, checkingOptions) {
+export async function checkFileContents(username, languageCode, repoCode, branch, filename, fileContent, givenLocation, checkingOptions) {
   // Determine the file type from the filename extension
   //  and return the results of checking that kind of file text
-  // console.log(`checkFileContents(${languageCode}, ${filename}, ${fileContent.length} chars, ${givenLocation}, ${JSON.stringify(checkingOptions)})…`);
+  // console.log(`checkFileContents(${username}, ${languageCode}, ${filename}, ${fileContent.length} chars, ${givenLocation}, ${JSON.stringify(checkingOptions)})…`);
+  console.assert(username !== undefined, "checkFileContents: 'username' parameter should be defined");
+  console.assert(typeof username === 'string', `checkFileContents: 'username' parameter should be a string not a '${typeof username}': ${username}`);
   console.assert(languageCode !== undefined, "checkFileContents: 'languageCode' parameter should be defined");
-  console.assert(typeof languageCode === 'string', `checkFileContents: 'languageCode' parameter should be a string not a '${typeof languageCode}'`);
+  console.assert(typeof languageCode === 'string', `checkFileContents: 'languageCode' parameter should be a string not a '${typeof languageCode}': ${languageCode}`);
   console.assert(repoCode !== undefined, "checkFileContents: 'repoCode' parameter should be defined");
-  console.assert(typeof repoCode === 'string', `checkFileContents: 'repoCode' parameter should be a string not a '${typeof repoCode}'`);
+  console.assert(typeof repoCode === 'string', `checkFileContents: 'repoCode' parameter should be a string not a '${typeof repoCode}': ${repoCode}`);
+  console.assert(branch !== undefined, "checkFileContents: 'branch' parameter should be defined");
+  console.assert(typeof branch === 'string', `checkFileContents: 'branch' parameter should be a string not a '${typeof branch}': ${branch}`);
   console.assert(filename !== undefined, "checkFileContents: 'filename' parameter should be defined");
-  console.assert(typeof filename === 'string', `checkFileContents: 'filename' parameter should be a string not a '${typeof filename}'`);
+  console.assert(typeof filename === 'string', `checkFileContents: 'filename' parameter should be a string not a '${typeof filename}': ${filename}`);
   console.assert(fileContent !== undefined, "checkFileContents: 'fileContent' parameter should be defined");
-  console.assert(typeof fileContent === 'string', `checkFileContents: 'fileContent' parameter should be a string not a '${typeof fileContent}'`);
-  console.assert(givenLocation !== undefined, "checkFileContents: 'givenRowLocation' parameter should be defined");
-  console.assert(typeof givenLocation === 'string', `checkFileContents: 'givenRowLocation' parameter should be a string not a '${typeof givenLocation}'`);
+  console.assert(typeof fileContent === 'string', `checkFileContents: 'fileContent' parameter should be a string not a '${typeof fileContent}': ${fileContent.length}`);
+  console.assert(givenLocation !== undefined, "checkFileContents: 'givenLocation' parameter should be defined");
+  console.assert(typeof givenLocation === 'string', `checkFileContents: 'givenLocation' parameter should be a string not a '${typeof givenLocation}': ${givenLocation}`);
   console.assert(checkingOptions !== undefined, "checkFileContents: 'checkingOptions' parameter should be defined");
 
   const startTime = new Date();
@@ -40,10 +47,10 @@ export async function checkFileContents(languageCode, repoCode, filename, fileCo
   if (filenameLower.endsWith('.tsv')) {
     const filenameMain = filename.substring(0, filename.length - 4); // drop .tsv
     // console.log(`checkFileContents have TSV filenameMain=${filenameMain}`);
-    const bookID = filenameMain.startsWith(`${languageCode}_`) ? filenameMain.substring(filenameMain.length - 3) : filenameMain.substring(0, 3).toUpperCase();
+    const bookID = filenameMain.startsWith(`${languageCode}_`) || filenameMain.startsWith('en_') ? filenameMain.substring(filenameMain.length - 3) : filenameMain.substring(0, 3).toUpperCase();
     // console.log(`checkFileContents have TSV bookID=${bookID}`);
     console.assert(bookID === 'OBS' || books.isValidBookID(bookID), `checkFileContents: '${bookID}' is not a valid USFM book identifier`);
-    if (filename.startsWith(`${languageCode}_`))
+    if (filename.startsWith(`${languageCode}_`) || filenameMain.startsWith('en_'))
       checkFileResult = await checkTN_TSVText(languageCode, bookID, filename, fileContent, ourCFLocation, checkingOptions);
     else {
       const annotationType = filenameMain.substring(4).toUpperCase();
@@ -69,7 +76,7 @@ export async function checkFileContents(languageCode, repoCode, filename, fileCo
   else if (filenameLower.endsWith('.txt'))
     checkFileResult = checkPlainText('text', filename, fileContent, ourCFLocation, checkingOptions);
   else if (filenameLower === 'manifest.yaml')
-    checkFileResult = await checkManifestText('', formRepoName(languageCode, repoCode), '', fileContent, ourCFLocation, checkingOptions); // don’t know username or branch
+    checkFileResult = await checkManifestText(username, formRepoName(languageCode, repoCode), branch, fileContent, ourCFLocation, checkingOptions); // don’t know username or branch
   else if (filenameLower.endsWith('.yaml'))
     checkFileResult = checkYAMLText(languageCode, filename, fileContent, ourCFLocation, checkingOptions);
   else {