Merge pull request #3735 from unfoldingWord-dev/feature-klappy-3186-a…

…lignmentFromVerseObjects

Feature #3186/Fix broken unit tests

__tests__/AlignmentHelpers.test.js

@@ -42,9 +42,9 @@ describe("Merge Alignment into Verse Objects", () => {
   it('handles contiguousAndNonContiguous', () => {
     mergeTest('contiguousAndNonContiguous');
   });
-  // it('handles titus 1:1', () => {
-  //   mergeTest('tit1:1');
-  // });
+  it('handles titus 1:1', () => {
+    mergeTest('tit1:1');
+  });
 });
 
 
@@ -126,7 +126,7 @@ const mergeTest = (name = {}) => {
 const unmergeTest = (name = {}) => {
   const json = readJSON(`${name}.json`);
   expect(json).toBeTruthy();
-  const {verseObjects, alignment, wordBank} = json;
-  const output = AlignmentHelpers.unmerge(verseObjects);
+  const {verseObjects, alignment, wordBank, alignedVerseString} = json;
+  const output = AlignmentHelpers.unmerge(verseObjects, alignedVerseString);
   expect(output).toEqual({alignment, wordBank});
 };

__tests__/fixtures/pivotAlignmentVerseObjects/tit1:1.json

@@ -1,5 +1,5 @@
 {
-  "alignedVerseString": "",
+  "alignedVerseString": "Παῦλος δοῦλος Θεοῦ ἀπόστολος δὲ Ἰησοῦ Χριστοῦ κατὰ πίστιν ἐκλεκτῶν Θεοῦ καὶ ἐπίγνωσιν ἀληθείας τῆς κατ’ εὐσέβειαν",
   "verseString": "Paul, a servant of God and an apostle of Jesus Christ, for the faith of God's chosen people and the knowledge of the truth that agrees with godliness,",
   "verseObjects": [
     {
@@ -299,34 +299,22 @@
       ]
     },
     {
-      "strongs": "G02250",
-      "lemma": "ἀλήθεια",
-      "morph": "Gr,N,,,,,GFS,",
+      "strongs": "G19220",
+      "lemma": "ἐπίγνωσις",
+      "morph": "Gr,N,,,,,AFS,",
       "occurrence": 1,
       "occurrences": 1,
       "tag": "k",
       "type": "milestone",
-      "content": "ἀληθείας",
+      "content": "ἐπίγνωσιν",
       "children": [
         {
           "tag": "w",
           "type": "word",
           "text": "the",
           "occurrence": 2,
           "occurrences": 3
-        }
-      ]
-    },
-    {
-      "strongs": "G19220",
-      "lemma": "ἐπίγνωσις",
-      "morph": "Gr,N,,,,,AFS,",
-      "occurrence": 1,
-      "occurrences": 1,
-      "tag": "k",
-      "type": "milestone",
-      "content": "ἐπίγνωσιν",
-      "children": [
+        },
         {
           "tag": "w",
           "type": "word",
@@ -356,16 +344,16 @@
         {
           "tag": "w",
           "type": "word",
-          "text": "truth",
-          "occurrence": 1,
-          "occurrences": 1
+          "text": "the",
+          "occurrence": 3,
+          "occurrences": 3
         },
         {
           "tag": "w",
           "type": "word",
-          "text": "the",
-          "occurrence": 3,
-          "occurrences": 3
+          "text": "truth",
+          "occurrence": 1,
+          "occurrences": 1
         }
       ]
     },
@@ -743,15 +731,15 @@
           "occurrence": 4,
           "occurrences": 4
         },
-        {
-          "word": "truth",
-          "occurrence": 1,
-          "occurrences": 1
-        },
         {
           "word": "the",
           "occurrence": 3,
           "occurrences": 3
+        },
+        {
+          "word": "truth",
+          "occurrence": 1,
+          "occurrences": 1
         }
       ]
     },

__tests__/usfmHelpers.test.js

@@ -109,29 +109,29 @@ describe('USFM Details', () => {
     const usfmFile = missingVerseMarkers;
     const usfmRaw = usfmHelpers.loadUSFMFile(usfmFile);
     const usfm = usfmHelpers.getParsedUSFM(usfmRaw).chapters;
-    expect(usfm[1][3]).toBeUndefined();
-    expect(usfm[1][4]).toBeUndefined();
-    expect(usfm[1][21]).toEqual(['For to me to live is Christ, and to die is gain.']);
-    expect(30 - Object.keys(usfm[1]).length).toEqual(2);
-
-    expect(usfm[2][5]).toBeUndefined();
-    expect(usfm[2][6]).toBeUndefined();
-    expect(usfm[2][10]).toEqual(['So at the name of Jesus every knee should bow,', "of those in heaven and on earth and under the earth."]);
-    expect(30 - Object.keys(usfm[2]).length).toEqual(2);
-
-    expect(usfm[3][5]).toBeUndefined();
-    expect(usfm[3][10]).toBeUndefined();
-    expect(usfm[3][11]).toBeUndefined();
-    expect(usfm[3][15]).toBeUndefined();
-    expect(usfm[3][16]).toBeUndefined();
-    expect(usfm[3][17]).toEqual(['Brothers, join me and imitate me, and watch closely those who are walking by our example.']);
-    expect(21 - Object.keys(usfm[3]).length).toEqual(5);
-
-    expect(usfm[4][1]).toBeUndefined();
-    expect(usfm[4][21]).toBeUndefined();
-    expect(usfm[4][22]).toBeUndefined();
-    expect(usfm[4][4]).toEqual(['Rejoice in the Lord always; again I will say, rejoice.']);
-    expect(23 - Object.keys(usfm[4]).length).toEqual(3);
+    expect(getVerseString(usfm,1,3)).toBeUndefined();
+    expect(getVerseString(usfm,1,4)).toBeUndefined();
+    expect(getVerseString(usfm,1,21)).toEqual('For to me to live is Christ, and to die is gain.');
+    expect(30 - getVerses(usfm[1]).length).toEqual(2);
+
+    expect(getVerseString(usfm,2,5)).toBeUndefined();
+    expect(getVerseString(usfm,2,6)).toBeUndefined();
+    expect(getVerseString(usfm,2,10)).toEqual('So at the name of Jesus every knee should bow, of those in heaven and on earth and under the earth.');
+    expect(30 - getVerses(usfm[2]).length).toEqual(2);
+
+    expect(getVerseString(usfm,3,5)).toBeUndefined();
+    expect(getVerseString(usfm,3,10)).toBeUndefined();
+    expect(getVerseString(usfm,3,11)).toBeUndefined();
+    expect(getVerseString(usfm,3,15)).toBeUndefined();
+    expect(getVerseString(usfm,3,16)).toBeUndefined();
+    expect(getVerseString(usfm,3,17)).toEqual('Brothers, join me and imitate me, and watch closely those who are walking by our example.');
+    expect(21 - getVerses(usfm[3]).length).toEqual(5);
+
+    expect(getVerseString(usfm,4,1)).toBeUndefined();
+    expect(getVerseString(usfm,4,21)).toBeUndefined();
+    expect(getVerseString(usfm,4,22)).toBeUndefined();
+    expect(getVerseString(usfm,4,4)).toEqual('Rejoice in the Lord always; again I will say, rejoice.');
+    expect(23 - getVerses(usfm[4]).length).toEqual(3);
   });
 
   test('should handle out of sequence verse markers', () => {
@@ -140,20 +140,20 @@ describe('USFM Details', () => {
     const usfmRaw = usfmHelpers.loadUSFMFile(usfmFile);
     const usfm = usfmHelpers.getParsedUSFM(usfmRaw).chapters;
     expect(usfm[1]).not.toBeUndefined();
-    expect(Object.keys(usfm[1])).toHaveLength(30);
-    expect(usfm[1][21]).toEqual(['For to me to live is Christ, and to die is gain.']);
+    expect(getVerses(usfm[1])).toHaveLength(30);
+    expect(getVerseString(usfm,1,21)).toEqual('For to me to live is Christ, and to die is gain.');
 
     expect(usfm[2]).not.toBeUndefined();
-    expect(Object.keys(usfm[2])).toHaveLength(30);
-    expect(usfm[2][10]).toEqual(['So at the name of Jesus every knee should bow,', "of those in heaven and on earth and under the earth."]);
+    expect(getVerses(usfm[2])).toHaveLength(30);
+    expect(getVerseString(usfm,2,10)).toEqual('So at the name of Jesus every knee should bow, of those in heaven and on earth and under the earth.');
 
     expect(usfm[3]).not.toBeUndefined();
-    expect(Object.keys(usfm[3])).toHaveLength(21);
-    expect(usfm[3][11]).toEqual(['so somehow I may experience the resurrection from the dead.']);
+    expect(getVerses(usfm[3])).toHaveLength(21);
+    expect(getVerseString(usfm,3,11)).toEqual('so somehow I may experience the resurrection from the dead.');
 
     expect(usfm[4]).not.toBeUndefined();
-    expect(Object.keys(usfm[4])).toHaveLength(23);
-    expect(usfm[4][4]).toEqual(['Rejoice in the Lord always; again I will say, rejoice.']);
+    expect(getVerses(usfm[4])).toHaveLength(23);
+    expect(getVerseString(usfm,4,4)).toEqual('Rejoice in the Lord always; again I will say, rejoice.');
   });
 
   test('should handle missing chapter markers', () => {
@@ -165,8 +165,8 @@ describe('USFM Details', () => {
     expect(usfm[2]).toBeUndefined();
 
     expect(usfm[3]).not.toBeUndefined();
-    expect(Object.keys(usfm[3])).toHaveLength(21);
-    expect(usfm[3][11]).toEqual(['so somehow I may experience the resurrection from the dead.']);
+    expect(getVerses(usfm[3])).toHaveLength(22);
+    expect(getVerseString(usfm,3,11)).toEqual('so somehow I may experience the resurrection from the dead.');
   });
 
   test('should handle missing a chapter marker', () => {
@@ -177,10 +177,10 @@ describe('USFM Details', () => {
     expect(usfm[2]).toBeUndefined();
     expect(usfm[3]).not.toBeUndefined();
 
-    expect(Object.keys(usfm[1])).toHaveLength(30);
-    expect(Object.keys(usfm[3])).toHaveLength(21);
-    expect(Object.keys(usfm[4])).toHaveLength(23);
-    expect(usfm[3][1]).toEqual(['Finally, my brothers, rejoice in the Lord.  For me to write these same things to you is not a problem, and they will keep you safe.']);
+    expect(getVerses(usfm[1])).toHaveLength(30);
+    expect(getVerses(usfm[3])).toHaveLength(21);
+    expect(getVerses(usfm[4])).toHaveLength(23);
+    expect(getVerseString(usfm,3,1)).toEqual('Finally, my brothers, rejoice in the Lord.  For me to write these same things to you is not a problem, and they will keep you safe.');
   });
 
   test('should handle out of sequence chapter markers', () => {
@@ -190,19 +190,54 @@ describe('USFM Details', () => {
     const usfm = usfmHelpers.getParsedUSFM(usfmRaw).chapters;
 
     expect(usfm[1]).not.toBeUndefined();
-    expect(Object.keys(usfm[1])).toHaveLength(30);
-    expect(usfm[1][21]).toEqual(['For to me to live is Christ, and to die is gain.']);
+    expect(getVerses(usfm[1])).toHaveLength(30);
+    expect(getVerseString(usfm,1,21)).toEqual('For to me to live is Christ, and to die is gain.');
 
     expect(usfm[2]).not.toBeUndefined();
-    expect(Object.keys(usfm[2])).toHaveLength(30);
-    expect(usfm[2][10]).toEqual(['So at the name of Jesus every knee should bow,', "of those in heaven and on earth and under the earth."]);
+    expect(getVerses(usfm[2])).toHaveLength(30);
+    expect(getVerseString(usfm,2,10)).toEqual('So at the name of Jesus every knee should bow, of those in heaven and on earth and under the earth.');
 
     expect(usfm[3]).not.toBeUndefined();
-    expect(Object.keys(usfm[3])).toHaveLength(21);
-    expect(usfm[3][11]).toEqual(['so somehow I may experience the resurrection from the dead.']);
+    expect(getVerses(usfm[3])).toHaveLength(21);
+    expect(getVerseString(usfm,3,11)).toEqual('so somehow I may experience the resurrection from the dead.');
 
     expect(usfm[4]).not.toBeUndefined();
-    expect(Object.keys(usfm[4])).toHaveLength(23);
-    expect(usfm[4][4]).toEqual(['Rejoice in the Lord always; again I will say, rejoice.']);
+    expect(getVerses(usfm[4])).toHaveLength(23);
+    expect(getVerseString(usfm,4,4)).toEqual('Rejoice in the Lord always; again I will say, rejoice.');
   });
+
+  //
+  // helpers
+  //
+
+  const getVerses = (verses) => {
+    if (verses.front) {
+      delete verses.front;
+    }
+    return Object.keys(verses);
+  };
+
+  const getVerseString = (chapters, chapterNum, verseNum) => {
+    let verse = chapters[chapterNum][verseNum];
+    if (verse) {
+      if (typeof verse === 'string') {
+        return verse.trim();
+      }
+      if (Array.isArray(verse)) {
+        return verse.join(' ').trim();
+      } else {
+        let retValue = '';
+        for (let object of verse.verseObjects) {
+          if ((object.type === 'text') || (object.type === 'quote')) {
+            if (retValue) {
+              retValue += ' ';
+            }
+            retValue += object.text.trim();
+          }
+        }
+        return retValue.trim();
+      }
+    }
+    return verse;
+  };
 });

package.json

@@ -133,7 +133,7 @@
     "simple-git": "1.43.0",
     "sudo-prompt": "6.2.1",
     "truncate-utf8-bytes": "1.0.2",
-    "usfm-js": "1.0.0-beta.21",
+    "usfm-js": "1.0.0-beta.22",
     "xregexp": "3.2.0",
     "yamljs": "0.3.0",
     "zip-folder": "1.0.0"

src/js/actions/TargetLanguageActions.js

@@ -5,6 +5,8 @@ import usfmjs from 'usfm-js';
 // helpers
 import * as USFMHelpers from '../helpers/usfmHelpers';
 import { getBibleIndex } from '../helpers/ResourcesHelpers';
+import * as VerseObjectHelpers from "../helpers/VerseObjectHelpers";
+
 // constant declarations
 const IMPORTED_SOURCE_PATH = '.apps/translationCore/importedSource';
 
@@ -60,7 +62,7 @@ export function generateTargetBibleFromUSFMPath(usfmFilePath, projectPath, manif
       targetBible[chapterNumber] = {};
       Object.keys(chapterObject).forEach((verseNumber)=>{
         const verseArray = chapterObject[verseNumber];
-        targetBible[chapterNumber][verseNumber] = verseArray.join(' ');
+        targetBible[chapterNumber][verseNumber] = VerseObjectHelpers.mergeVerseData(verseArray);
       });
     });
     saveTargetBible(projectPath, manifest, targetBible);
@@ -116,17 +118,21 @@ export function generateTargetBibleFromProjectPath(projectPath, manifest) {
             let chunkVerseNumber = parseInt(chunkFileNumber[1]);
             const chunkPath = path.join(chapterPath, file);
             let text = fs.readFileSync(chunkPath).toString();
-            if (!text.includes('\\v')) text = `\\v ${chunkVerseNumber} ` + text;
-            const currentChunk = usfmjs.toJSON(text, {chunk: true});
+            const hasChapters = text.includes('\\c ');
+            if (!text.includes('\\v')) {
+              text = `\\v ${chunkVerseNumber} ` + text;
+            }
+            const currentChunk = usfmjs.toJSON(text, {chunk: !hasChapters});
 
             if (currentChunk && currentChunk.chapters[chapterNumber]) {
-              Object.keys(currentChunk.chapters[chapterNumber]).forEach((key) => {
-                chapterData[key] = currentChunk.chapters[chapterNumber][key][0];
+              const chapter = currentChunk.chapters[chapterNumber];
+              Object.keys(chapter).forEach((key) => {
+                chapterData[key] =  VerseObjectHelpers.mergeVerseData(chapter[key]);
                 bookData[parseInt(chapterNumber)] = chapterData;
               });
             } else if (currentChunk && currentChunk.verses) {
               Object.keys(currentChunk.verses).forEach((key) => {
-                chapterData[key] = currentChunk.verses[key][0];
+                chapterData[key] =  VerseObjectHelpers.mergeVerseData(currentChunk.verses[key]);
                 bookData[parseInt(chapterNumber)] = chapterData;
               });
             }

src/js/actions/WordAlignmentLoadActions.js

@@ -101,8 +101,8 @@ export const generateBlankAlignments = (verseData) => {
     })
     .map((wordData, index) => {
       const word = wordData.word || wordData.text;
-      let occurrences = WordAlignmentHelpers.occurrencesInString(combinedVerse, word);
-      let occurrence = WordAlignmentHelpers.getOccurrenceInString(combinedVerse, index, word);
+      let occurrences = stringHelpers.occurrencesInString(combinedVerse, word);
+      let occurrence = stringHelpers.occurrenceInString(combinedVerse, index, word);
       const alignment = {
         topWords: [
           {
@@ -129,8 +129,8 @@ export const generateWordBank = (verseText) => {
   // TODO: remove once occurrencesInString uses tokenizer, can't do that until bug is addressed with Greek
   const _verseText = verseWords.join(' ');
   const wordBank = verseWords.map((word, index) => {
-    let occurrences = WordAlignmentHelpers.occurrencesInString(_verseText, word);
-    let occurrence = WordAlignmentHelpers.getOccurrenceInString(_verseText, index, word);
+    let occurrences = stringHelpers.occurrencesInString(_verseText, word);
+    let occurrence = stringHelpers.occurrenceInString(_verseText, index, word);
     return {
       word,
       occurrence,