Skip to content

Commit

Permalink
Improve preloading code for new format branches
Browse files Browse the repository at this point in the history
  • Loading branch information
RobH123 committed Nov 22, 2020
1 parent 2e86120 commit 475c116
Show file tree
Hide file tree
Showing 9 changed files with 166 additions and 155 deletions.
14 changes: 7 additions & 7 deletions src/__tests__/__snapshots__/book-package-check.test.js.snap
Original file line number Diff line number Diff line change
Expand Up @@ -7021,21 +7021,21 @@ Object {
},
Object {
"details": "username=unfoldingWord",
"extra": "TN1",
"location": " in TN1 in zzz TIT book package from unfoldingWord master branch",
"extra": "TN",
"location": " in TN in zzz TIT book package from unfoldingWord master branch",
"message": "Repository doesn't exist",
"priority": 997,
"repoCode": "TN1",
"repoCode": "TN",
"repoName": "zzz_tn",
"username": "unfoldingWord",
},
Object {
"details": "username=unfoldingWord",
"extra": "TQ1",
"extra": "TQ",
"location": " in unfoldingWord (master)",
"message": "Repository doesn't exist",
"priority": 997,
"repoCode": "TQ1",
"repoCode": "TQ",
"repoName": "zzz_tq",
"username": "unfoldingWord",
},
Expand Down Expand Up @@ -7078,9 +7078,9 @@ Object {
Object {
"bookID": "TIT",
"details": "username=unfoldingWord",
"extra": "TN1",
"extra": "TN",
"filename": "zzz_tn_57-TIT.tsv",
"location": " in TN1 in zzz TIT book package from unfoldingWord master branch: Could not find src/__tests__/fixtures/unfoldingWord/zzz_tn/zzz_tn_57-TIT.tsv",
"location": " in TN in zzz TIT book package from unfoldingWord master branch: Could not find src/__tests__/fixtures/unfoldingWord/zzz_tn/zzz_tn_57-TIT.tsv",
"message": "Failed to load",
"priority": 996,
"repoName": "zzz_tn",
Expand Down
20 changes: 12 additions & 8 deletions src/core/getApi.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import * as books from './books';
import { clearCheckedArticleCache } from './tn-links-check';


// const GETAPI_VERSION_STRING = '0.6.4';
// const GETAPI_VERSION_STRING = '0.6.5';

const MAX_INDIVIDUAL_FILES_TO_DOWNLOAD = 5; // More than this and it downloads the zipfile for the entire repo

Expand Down Expand Up @@ -227,20 +227,20 @@ export async function cachedGetBookFilenameFromManifest({ username, repository,
* @param {string} languageCode
* @param {Array} bookIDList - one or more books that will be checked
* @param {string} branch - optional, defaults to master
* @param {Array} repos - optional, list of repos to pre-load
* @param {Array} repoList - optional, list of repos to pre-load
* @return {Promise<Boolean>} resolves to true if file loads are successful
*/
export async function preloadReposIfNecessary(username, languageCode, bookIDList, branch = 'master', repos = ['TA', 'TW', languageCode === 'en' ? 'TQ2' : 'TQ']) {
export async function preloadReposIfNecessary(username, languageCode, bookIDList, branch, repoList) {
// NOTE: We preload TA and TW by default because we are likely to have many links to those repos
// We preload TQ by default because it has thousands of files (17,337), so individual file fetches might be slow
// even for one book which might have several hundred files.
console.log(`preloadReposIfNecessary(${username}, ${languageCode}, ${bookIDList} (${typeof bookID}), ${branch}, [${repos}])…`);
console.log(`preloadReposIfNecessary(${username}, ${languageCode}, ${bookIDList} (${typeof bookID}), ${branch}, [${repoList}])…`);
let success = true;

const repos_ = [...repos];
const repos_ = [...repoList];
if (bookIDList.length === 1 && bookIDList[0] === 'OBS') {
if (!repos_.includes('OBS'))
repos_.unshift('OBS');
repos_.unshift('OBS'); // push to beginning of list
}
if (bookIDList && Array.isArray(bookIDList) && bookIDList.length > MAX_INDIVIDUAL_FILES_TO_DOWNLOAD) { // Fetch individually if checking less books
// make sure we have the original languages needed
Expand Down Expand Up @@ -284,9 +284,13 @@ export async function preloadReposIfNecessary(username, languageCode, bookIDList
let adjustedLanguageCode = languageCode;
if ((languageCode === 'hbo' && repoCode !== 'UHB') || (languageCode === 'el-x-koine' && repoCode !== 'UGNT'))
adjustedLanguageCode = 'en'; // Assume English then
const repoName = formRepoName(adjustedLanguageCode, repoCode);
let adjustedBranch = branch;
let adjustedRepoCode = repoCode;
if (repoCode === 'TQ2') { adjustedRepoCode = 'TQ'; adjustedBranch = 'newFormat'; }
else if (repoCode === 'TN2') { adjustedRepoCode = 'TN'; adjustedBranch = 'newFormat'; }
const repoName = formRepoName(adjustedLanguageCode, adjustedRepoCode);
// console.log(`preloadReposIfNecessary: preloading zip file for ${repoName}…`);
const zipFetchSucceeded = await cachedGetRepositoryZipFile({ username, repository: repoName, branch });
const zipFetchSucceeded = await cachedGetRepositoryZipFile({ username, repository: repoName, branch: adjustedBranch });
if (!zipFetchSucceeded) {
console.error(`preloadReposIfNecessary() misfetched zip file for ${repoCode} repo with ${zipFetchSucceeded}`);
success = false;
Expand Down
6 changes: 3 additions & 3 deletions src/demos/all-book-packages-check/AllBookPackagesCheck.js
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ function AllBookPackagesCheck(/*username, languageCode, bookIDs,*/ props) {
// Or this allows the parameters to be specified as a BookPackagesCheck property
if (props.extractLength) checkingOptions.extractLength = ourParseInt(props.extractLength);

let preloadList = ['TA', 'TW', 'TQ2'];
if (bookIDList.length > 5) { preloadList.push('LT'); preloadList.push('ST'); preloadList.push('TN2'); }
let repoPreloadList = ['TA', 'TW', 'TQ', 'TQ2'];
if (bookIDList.length > 5) { repoPreloadList.push('LT'); repoPreloadList.push('ST'); repoPreloadList.push('TN'); repoPreloadList.push('TN2'); }

useEffect(() => {
// console.log("BookPackagesCheck.useEffect() called with ", JSON.stringify(props));
Expand All @@ -93,7 +93,7 @@ function AllBookPackagesCheck(/*username, languageCode, bookIDs,*/ props) {
else await clearCheckedArticleCache();

setResultValue(<p style={{ color: 'magenta' }}>Preloading repos for {username} {languageCode} ready for all book packages check…</p>);
const successFlag = await preloadReposIfNecessary(username, languageCode, bookIDList, branch, preloadList);
const successFlag = await preloadReposIfNecessary(username, languageCode, bookIDList, branch, repoPreloadList);
if (!successFlag)
console.log(`AllBookPackagesCheck error: Failed to pre-load all repos`)

Expand Down
4 changes: 3 additions & 1 deletion src/demos/book-package-check/BookPackageCheck.js
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ function BookPackageCheck(/*username, languageCode, bookID,*/ props) {
// console.log(`checkingOptions.checkLinkedTAArticleFlag ${checkingOptions.checkLinkedTAArticleFlag} from '${props.checkLinkedTAArticleFlag}'`);
// console.log(`checkingOptions.checkLinkedTWArticleFlag ${checkingOptions.checkLinkedTWArticleFlag} from '${props.checkLinkedTWArticleFlag}'`);

const repoPreloadList = ['TA', 'TW', 'TQ', 'TQ2'];

useEffect(() => {
// const newProps = { bookID, branch, checkingOptions, languageCode, cutoffPriorityLevel: props.cutoffPriorityLevel, displayType: props.displayType, errorPriorityLevel: props.errorPriorityLevel, maximumSimilarMessages: props.maximumSimilarMessages, sortBy: props.sortBy, username};
// console.log("BookPackageCheck.useEffect() called with ", JSON.stringify(newProps));
Expand Down Expand Up @@ -81,7 +83,7 @@ function BookPackageCheck(/*username, languageCode, bookID,*/ props) {

// if (bookID !== 'OBS') { // Preload the reference repos
setResultValue(<p style={{ color: 'magenta' }}>Preloading repos for {username} {languageCode} ready for <b>{bookID}</b> book package check…</p>);
const successFlag = await preloadReposIfNecessary(username, languageCode, [bookID], branch);
const successFlag = await preloadReposIfNecessary(username, languageCode, [bookID], branch, repoPreloadList);
if (!successFlag)
console.error(`BookPackageCheck error: Failed to pre-load all repos`)
// }
Expand Down
6 changes: 3 additions & 3 deletions src/demos/book-packages-check/BookPackagesCheck.js
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ function BookPackagesCheck(/*username, languageCode, bookIDs,*/ props) {
// Or this allows the parameters to be specified as a BookPackagesCheck property
if (props.extractLength) checkingOptions.extractLength = ourParseInt(props.extractLength);

let preloadList = ['TA', 'TW', 'TQ2'];
if (bookIDList.length > 5) { preloadList.push('LT'); preloadList.push('ST'); preloadList.push('TN2'); }
let repoPreloadList = ['TA', 'TW', 'TQ', 'TQ2'];
if (bookIDList.length > 5) { repoPreloadList.push('LT'); repoPreloadList.push('ST'); repoPreloadList.push('TN'); repoPreloadList.push('TN2'); }

useEffect(() => {
// console.log("BookPackagesCheck.useEffect() called with ", JSON.stringify(props));
Expand All @@ -71,7 +71,7 @@ function BookPackagesCheck(/*username, languageCode, bookIDs,*/ props) {
else await clearCheckedArticleCache();

setResultValue(<p style={{ color: 'magenta' }}>Preloading repos for {username} {languageCode} ready for book packages check…</p>);
const successFlag = await preloadReposIfNecessary(username, languageCode, bookIDList, branch, preloadList);
const successFlag = await preloadReposIfNecessary(username, languageCode, bookIDList, branch, repoPreloadList);
if (!successFlag)
console.error(`BookPackagesCheck error: Failed to pre-load all repos`)

Expand Down
4 changes: 3 additions & 1 deletion src/demos/gl-book-package-check/GlBookPackageCheck.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ function GlBookPackageCheck(/*username, languageCode, bookIDs,*/ props) {
// Or this allows the parameters to be specified as a GlBookPackageCheck property
if (props.extractLength) checkingOptions.extractLength = ourParseInt(props.extractLength);

const repoPreloadList = ['TA', 'TW', 'TQ', 'TQ2'];

useEffect(() => {
// console.log("GlBookPackageCheck.useEffect() called with ", JSON.stringify(props));

Expand Down Expand Up @@ -72,7 +74,7 @@ function GlBookPackageCheck(/*username, languageCode, bookIDs,*/ props) {
else await clearCheckedArticleCache();

setResultValue(<p style={{ color: 'magenta' }}>Preloading repos for {username} {languageCode} ready for GL book package check…</p>);
const successFlag = await preloadReposIfNecessary(username, languageCode, [bookID], branch);
const successFlag = await preloadReposIfNecessary(username, languageCode, [bookID], branch, repoPreloadList);
if (!successFlag)
console.error(`AllBookPackagesCheck error: Failed to pre-load all repos`)

Expand Down
3 changes: 2 additions & 1 deletion src/demos/notice-processing-functions.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { isDisabledNotice } from './disabled-notices';
// import { displayPropertyNames, consoleLogObject } from './utilities';


// const NOTICE_PROCESSOR_VERSION_STRING = '0.8.9';
// const NOTICE_PROCESSOR_VERSION_STRING = '0.8.10';

// All of the following can be overriden with optionalProcessingOptions
const DEFAULT_MAXIMUM_SIMILAR_MESSAGES = 3; // Zero means no suppression of similar messages
Expand Down Expand Up @@ -115,6 +115,7 @@ function processNoticesCommon(givenNoticeObject, optionalProcessingOptions) {
&& (thisUniqueNotice.rowID === item.rowID || thisUniqueNotice.rowID === undefined || item.rowID === undefined)
&& (thisUniqueNotice.lineNumber === item.lineNumber || thisUniqueNotice.lineNumber === undefined || item.lineNumber === undefined)
&& (thisUniqueNotice.characterIndex === item.characterIndex || thisUniqueNotice.characterIndex === undefined || item.characterIndex === undefined)
&& (thisUniqueNotice.extract === item.extract || thisUniqueNotice.extract === undefined || item.extract === undefined)
)
return ix;
}
Expand Down
8 changes: 4 additions & 4 deletions src/demos/repo-check/RepoCheck.js
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,10 @@ function RepoCheck(/*username, languageCode,*/ props) {
// console.log(`RepoCheck languageCode='${languageCode}' repoCode='${repoCode}'`);

setResultValue(<p style={{ color: 'magenta' }}>Preloading repos for {username} {languageCode} ready for {repoName} repo check…</p>);
const repoList = ['TW'];
if (repoCode !== 'UHB' && repoCode !== 'UGNT') repoList.push('TA'); // Original languages only have TW links
if (repoCode !== 'TA' && repoCode !== 'TW') repoList.push(repoCode);
const successFlag = await preloadReposIfNecessary(username, languageCode, [], branch, repoList);
const repoPreloadList = ['TW'];
if (repoCode !== 'UHB' && repoCode !== 'UGNT') repoPreloadList.push('TA'); // Original languages only have TW links
if (repoCode !== 'TA' && repoCode !== 'TW') repoPreloadList.push(repoCode);
const successFlag = await preloadReposIfNecessary(username, languageCode, [], branch, repoPreloadList);
if (!successFlag)
console.error(`RepoCheck error: Failed to pre-load all repos`)

Expand Down
Loading

0 comments on commit 475c116

Please sign in to comment.