More fine-tuning of link checking

noticeList.txt

@@ -1,4 +1,4 @@
-Last updated 2021-03-03 08:50:12.058433 by makeNoticeList.py
+Last updated 2021-03-03 14:21:40.746190 by makeNoticeList.py
 Got 496 notices:
   checkRepoResult.noticeList.push( 997, "Repository doesn’t exist", details, username, repoCode, repoName, location: givenLocation, extra: repoCode	from checkRepo.js line 256
   languageCode === 'en' || languageCode === 'fr' ? 490 : 190, "Expected header field to contain a mixed-case string", fieldName: `\\$marker`, excerpt: rest, C, V, location: lineLocation	from usfm-text-check.js line 945
@@ -24,7 +24,7 @@ Got 496 notices:
   thisPriority, thisMessage, excerpt: regexResultArray[0], location: ourLocation	from field-text-check.js line 417
   const notice =  252, "Markdown headers should be preceded by a blank line", lineNumber: n, location: ourLocation ;	from markdown-text-check.js line 281
   notice	from markdown-text-check.js line 284
-  const notice =  252, "Markdown headers should be followed by a blank line", lineNumber: n, location: ourLocation ;	from markdown-text-check.js line 287
+  const notice =  251, "Markdown headers should be followed by a blank line", lineNumber: n, location: ourLocation ;	from markdown-text-check.js line 287
   notice	from markdown-text-check.js line 290
   const notice =  172, "Header levels should only increment by one", lineNumber: n, characterIndex: 0, location: ourLocation ;	from markdown-text-check.js line 300
   notice	from markdown-text-check.js line 303
@@ -491,7 +491,7 @@ Got 496 notices:
   64, "Unexpected leading space(s) after break", characterIndex, excerpt, location: ourLocation	from field-text-check.js line 155
   63, "Unexpected leading space(s) after line break", characterIndex, excerpt, location: ourLocation	from field-text-check.js line 160
   50, "Is this quote/occurrence correct???", details: `Occurrence=$occurrence`, excerpt: fieldText, location: ourLocation	from orig-quote-check.js line 305
-  32, `Untested general link`, details: "please manually double-check link—probably no problem", excerpt: totalLink, location: ourLocation	from notes-links-check.js line 1,092
+  32, `Untested general/outside link`, details: "please manually double-check link—probably no problem", excerpt: totalLink, location: ourLocation	from notes-links-check.js line 1,092
   20, "Note that 'disableAllLinkFetchingFlag' was set so link targets were not checked", location: ourLocation	from tn-tsv9-table-check.js line 263
   20, "Note that 'disableAllLinkFetchingFlag' was set so link targets were not checked", location: ourLocation	from twl-tsv6-table-check.js line 252
   20, "Note that 'disableAllLinkFetchingFlag' was set so link targets were not checked", location: ourLocation	from notes-tsv7-table-check.js line 252

package.json

@@ -1,7 +1,7 @@
 {
   "name": "uw-content-validation",
   "description": "Functions for Checking Door43.org Scriptural Content/Resources.",
-  "version": "2.0.0_alpha11",
+  "version": "2.0.0",
   "private": false,
   "homepage": "https://unfoldingword.github.io/uw-content-validation/",
   "repository": {

src/__tests__/__snapshots__/book-package-check.test.js.snap

@@ -27139,7 +27139,7 @@ Object {
       "lineNumber": 2,
       "location": " in en TIT book package from unfoldingWord master branch",
       "message": "Markdown headers should be followed by a blank line",
-      "priority": 252,
+      "priority": 251,
       "repoCode": "SQ",
       "repoName": "en_sq",
       "rowID": "gtn1",

src/core/markdown-text-check.js

@@ -7,7 +7,7 @@ import { removeDisabledNotices } from './disabled-notices';
 import { parameterAssert, dataAssert, debugLog } from './utilities';
 
 
-const MARKDOWN_TEXT_VALIDATOR_VERSION_STRING = '0.7.0';
+const MARKDOWN_TEXT_VALIDATOR_VERSION_STRING = '0.7.1';
 
 
 /**
@@ -284,7 +284,7 @@ export async function checkMarkdownText(languageCode, repoCode, textOrFileName,
                 addNotice(notice);
             }
             if (nextLine?.length !== 0) {
-                const notice = { priority: 252, message: "Markdown headers should be followed by a blank line", lineNumber: n, location: ourLocation };
+                const notice = { priority: 251, message: "Markdown headers should be followed by a blank line", lineNumber: n, location: ourLocation };
                 if (textOrFileName === 'Note' || textOrFileName === 'OccurrenceNote')
                     notice.details = `markdown line ${n}`;
                 addNotice(notice);

src/core/notes-links-check.js

@@ -8,7 +8,7 @@ import { userLog, debugLog, functionLog, parameterAssert, logicAssert, dataAsser
 // import { consoleLogObject } from '../core/utilities';
 
 
-// const NOTES_LINKS_VALIDATOR_VERSION_STRING = '0.7.12';
+// const NOTES_LINKS_VALIDATOR_VERSION_STRING = '0.7.14';
 
 // const DEFAULT_LANGUAGE_CODE = 'en';
 const DEFAULT_BRANCH = 'master';
@@ -22,7 +22,7 @@ const TA_RELATIVE_DISPLAY_LINK_REGEX = new RegExp('\\[([^\\]]+?)\\]\\(\\.{2}/([^
 
 const TW_DOUBLE_BRACKETED_LINK_REGEX = new RegExp('\\[\\[rc://([^ /]+?)/tw/dict/bible/([^ /]+?)/([^ /\\]]+?)\\]\\]', 'g'); // Enclosed in [[  ]]
 const TWL_RAW_LINK_REGEX = new RegExp('rc://([^ /]+?)/tw/dict/bible/([^ /]+?)/(.+)', 'g'); // Just a raw link
-const TW_INTERNAL_REGEX = new RegExp('\\[([A-za-z]+?)\\]\\(\\.{2}/([a-z]{2,5})/([-A-Za-z]{2,20})\\.md\\)', 'g');// [Asher](../names/asher.md)
+const TW_INTERNAL_REGEX = new RegExp('\\[([A-za-z ()]+?)\\]\\(\\.{2}/([a-z]{2,5})/([-A-Za-z\\d]{2,20})\\.md\\)', 'g');// [Asher](../names/asher.md)
 
 // TODO: Do we need to normalise Bible links, i.e., make sure that the link itself
 //          (we don't care about the displayed text) doesn't specify superfluous levels/information
@@ -37,7 +37,7 @@ const THIS_VERSE_TO_THIS_CHAPTER_BIBLE_REGEX = new RegExp('\\[(\\d{1,3})\\]\\(\\
 const THIS_VERSE_RANGE_TO_THIS_CHAPTER_BIBLE_REGEX = new RegExp('\\[(\\d{1,3})[–-](\\d{1,3})\\]\\(\\.{2}/(\\d{1,3})/(\\d{1,3})\\.md\\)', 'g');// [2–7](../09/2.md) NOTE en-dash
 const BCV_V_TO_THIS_CHAPTER_BIBLE_REGEX = new RegExp('\\[((?:1 |2 |3 )?)((?:\\w+? )?)(\\d{1,3}):(\\d{1,3})[–-](\\d{1,3})\\]\\(\\./(\\d{1,3})\\.md\\)', 'g'); // [Genesis 26:12-14](./12.md) NOTE en-dash
 
-const BIBLE_FULL_HELP_REGEX = new RegExp('\\[((?:1 |2 |3 )?)((?:\\w+? )?)(\\d{1,3}):(\\d{1,3})(?:-\\d{1,3})\\]\\(rc://([^ /]+?)/tn/help/([123a-z]{3})/(\\d{1,3})/(\\d{1,3})\\)', 'g'); // [Genesis 29:23-24](rc://en/tn/help/gen/29/23)
+const BIBLE_FULL_HELP_REGEX = new RegExp('\\[((?:1 |2 |3 )?)((?:\\w+? )?)(\\d{1,3}):(\\d{1,3})(?:-\\d{1,3})?\\]\\(rc://([^ /]+?)/tn/help/([123a-z]{3})/(\\d{1,3})/(\\d{1,3})\\)', 'g'); // [Genesis 29:23-24](rc://en/tn/help/gen/29/23)
 
 const TN_REGEX = new RegExp('\\[((?:1 |2 |3 )?)((?:\\w+? )?)(\\d{1,3}):(\\d{1,3})\\]\\((\\.{2})/(\\d{1,3})/(\\d{1,3})/([a-z][a-z0-9][a-z0-9][a-z0-9])\\)', 'g');
 
@@ -1087,9 +1087,9 @@ export async function checkNotesLinksToOutside(languageCode, repoCode, bookID, g
                 // debugLog(`checkNotesLinksToOutside general link check needs to check: ${uri}`);
                 const serverString = uri.replace('://','!!!').split('/')[0].replace('!!!','://').toLowerCase(); // Get the bit before any forward slashes
 
-                // TODO: Uncomment this block and get it working better
-                if (!serverString.endsWith('door43.org') && !serverString.endsWith('unfoldingword.org')) // don't try to fetch general links
-                    addNoticePartial({ priority: 32, message: `Untested general link`, details: "please manually double-check link—probably no problem", excerpt: totalLink, location: ourLocation });
+                // NOTE: These message strings must match RenderProcessedResults.js
+                if (!serverString.endsWith('door43.org') && !serverString.endsWith('unfoldingword.org') && !serverString.endsWith('ufw.io')) // don't try to fetch general links
+                    addNoticePartial({ priority: 32, message: `Untested general/outside link`, details: "please manually double-check link—probably no problem", excerpt: totalLink, location: ourLocation });
                 else { // Try to fetch general links
                     let generalFileContent, hadError = false;
                     try {

src/demos/RenderProcessedResults.js

@@ -1,6 +1,7 @@
 import React from 'react';
 import { forwardRef } from 'react';
-import { parameterAssert, userLog } from '../core/utilities';
+// eslint-disable-next-line no-unused-vars
+import { parameterAssert, userLog, debugLog } from '../core/utilities';
 
 // NOTE: The following line is currently giving compile warnings -- a problem in a dependency it seems
 import MaterialTable from 'material-table';
@@ -41,7 +42,7 @@ const tableIcons = {
 };
 
 
-// const RENDER_PROCESSED_RESULTS_VERSION = '0.5.11';
+// const RENDER_PROCESSED_RESULTS_VERSION = '0.6.0';
 
 
 export function RenderSuccesses({ username, results }) {
@@ -257,7 +258,7 @@ function RenderFileDetails({ username, repoName, branch, filename, lineNumber, r
     // debugLog(`RenderFileDetails(${repoName}, ${branch}, ${filename}, ${lineNumber}, ${rowID}, ${fieldName})`);
     if (!repoName && !filename && !lineNumber && !rowID && !fieldName)
         return null; // They're all undefined or blank!
-    if (!branch) branch = repoName?.endsWith('2')? 'newFormat':'master'; // default but with TEMP code for newFormat
+    if (!branch) branch = repoName?.endsWith('2') ? 'newFormat' : 'master'; // default but with TEMP code for newFormat
     // debugLog(`RenderFileDetails2 ${repoName}, ${filename}, ${lineNumber}`);
     let resultStart = '', lineResult = '', resultEnd = '', fileLineLink = '', fileLink = '';
     if (repoName && repoName.length) resultStart += ` in ${repoName} repository`;
@@ -287,6 +288,29 @@ function RenderFileDetails({ username, repoName, branch, filename, lineNumber, r
     else if (fileLink) return <>{resultStart} in file <a rel="noopener noreferrer" target="_blank" href={fileLink}>{filename}</a>{resultEnd}</>;
     else return <>{resultStart}<b>{lineResult}</b>{resultEnd}</>;
 }
+// end of RenderFileDetails
+
+function RenderExcerpt({ excerpt, message }) {
+    // debugLog(`RenderExcerpt(${excerpt}, ${message})`);
+    // NOTE: These message strings must match notes-links-check.js (priority 82, and priority 32,)
+    // Note that messages might start with a repo code, e.g., "TN Actual message start"
+    if (message.endsWith("Untested general/outside link")
+        || message.endsWith("Error loading general link")
+        || message.endsWith("Should http link be https")) {
+        // debugLog(`Here1 RenderExcerpt(${excerpt}, ${message})`);
+        if (excerpt && excerpt[0] === '[' && excerpt.slice(-1) === ')') {
+            // debugLog(`Here2 RenderExcerpt(${excerpt}, ${message})`);
+            const ix = excerpt.indexOf('](');
+            const displayPart = excerpt.substring(1, ix); // Start after the [ unril before the ](
+            const linkPart = excerpt.substring(ix+2, excerpt.length-1); // Step past the ]( but don't include the final )
+            const adjLinkPart = message === "Should http link be https" ? linkPart.replace('http:', 'https:') : linkPart;
+            // debugLog(`RenderExcerpt from '${excerpt}' got ix=${ix}, displayPart='${displayPart}', linkPart='${linkPart}', adjLinkPart='${adjLinkPart}'`);
+            return <><span style={{ color: 'DimGray' }}>` around ►[${displayPart}](<a rel="noopener noreferrer" target="_blank" href={adjLinkPart}>{linkPart}</a>)◄`</span></>
+        }
+    }
+    return <><span style={{ color: 'DimGray' }}>{excerpt ? ` around ►${excerpt}◄` : ""}</span></>
+}
+// end of RenderExcerpt
 
 function RenderSuccessesColored({ results }) {
     // Display our array of success message strings in a nicer format
@@ -347,7 +371,7 @@ function RenderProcessedArray({ arrayType, results }) {
                     <RenderBCV bookID={listEntry.bookID} C={listEntry.C} V={listEntry.V} />
                     <RenderFileDetails username={listEntry.username} repoName={listEntry.repoName} branch={listEntry.branch} filename={listEntry.filename} lineNumber={listEntry.lineNumber} rowID={listEntry.rowID} fieldName={listEntry.fieldName} />
                     {listEntry.characterIndex > 0 ? " (at character " + (listEntry.characterIndex + 1) + ")" : ""}
-                    <span style={{ color: 'DimGray' }}>{listEntry.excerpt ? ` around ►${listEntry.excerpt}◄` : ""}</span>
+                    <RenderExcerpt excerpt={listEntry.excerpt} message={listEntry.message} />
                     {listEntry.location}
                     <RenderPriority entry={listEntry} />
                 </li>;
@@ -375,7 +399,7 @@ function RenderGivenArray({ array, color }) {
                 <RenderBCV bookID={listEntry.bookID} C={listEntry.C} V={listEntry.V} />
                 <RenderFileDetails username={listEntry.username} repoName={listEntry.repoName} branch={listEntry.branch} filename={listEntry.filename} lineNumber={listEntry.lineNumber} rowID={listEntry.rowID} fieldName={listEntry.fieldName} />
                 {listEntry.characterIndex !== undefined && listEntry.characterIndex >= 0 ? " (at character " + (listEntry.characterIndex + 1) + " of line)" : ""}
-                <span style={{ color: 'DimGray' }}>{listEntry.excerpt ? ` around ►${listEntry.excerpt}◄` : ""}</span>
+                <RenderExcerpt excerpt={listEntry.excerpt} message={listEntry.message} />
                 {listEntry.location}
                 <RenderPriority entry={listEntry} />
             </li>;
@@ -413,7 +437,7 @@ function RenderWarningsGradient({ results }) {
                 <RenderBCV bookID={listEntry.bookID} C={listEntry.C} V={listEntry.V} />
                 <RenderFileDetails username={listEntry.username} repoName={listEntry.repoName} branch={listEntry.branch} filename={listEntry.filename} lineNumber={listEntry.lineNumber} rowID={listEntry.rowID} fieldName={listEntry.fieldName} />
                 {listEntry.characterIndex !== undefined && listEntry.characterIndex >= 0 ? " (at character " + (listEntry.characterIndex + 1) + " of line)" : ""}
-                <span style={{ color: 'DimGray' }}>{listEntry.excerpt ? ` around ►${listEntry.excerpt}◄` : ""}</span>
+                <RenderExcerpt excerpt={listEntry.excerpt} message={listEntry.message} />
                 {listEntry.location}
                 <RenderPriority entry={listEntry} />
             </li>;