add: parsing worker

fix: #1166

package.json

@@ -56,7 +56,7 @@
     "unist-util-visit": "^5.0.0",
     "unist-util-visit-parents": "^6.0.1",
     "yamljs": "^0.3.0",
-    "zotero-plugin-toolkit": "^4.0.6"
+    "zotero-plugin-toolkit": "^4.0.7"
   },
   "devDependencies": {
     "@esbuild-plugins/node-globals-polyfill": "^0.2.3",

src/addon.ts

@@ -5,6 +5,8 @@ import { SyncDataType } from "./modules/sync/managerWindow";
 import hooks from "./hooks";
 import api from "./api";
 import { createZToolkit } from "./utils/ztoolkit";
+import { MessageHelper } from "zotero-plugin-toolkit/dist/helpers/message";
+import type { handlers } from "./extras/parsingWorker";
 
 class Addon {
   public data: {
@@ -70,6 +72,9 @@ class Addon {
     relation: {
       worker?: Worker;
     };
+    parsing: {
+      server?: MessageHelper<typeof handlers>;
+    };
     imageCache: Record<number, string>;
     hint: {
       silent: boolean;
@@ -117,6 +122,7 @@ class Addon {
       },
     },
     relation: {},
+    parsing: {},
     imageCache: {},
     hint: {
       silent: false,

src/elements/linkCreator/outlinePicker.ts

@@ -131,7 +131,9 @@ export class OutlinePicker extends PluginCEBase {
     if (!this.item) {
       return;
     }
-    this.noteOutline = this._addon.api.note.getNoteTreeFlattened(this.item);
+    this.noteOutline = await this._addon.api.note.getNoteTreeFlattened(
+      this.item,
+    );
     // Fake a cursor position
     if (typeof this.lineIndex === "number") {
       // @ts-ignore - formatValues is not in the types

src/elements/workspace/outlinePane.ts

@@ -211,12 +211,13 @@ export class OutlinePane extends PluginCEBase {
       "message",
       this.messageHandler,
     );
+    const nodes = await this._addon.api.note.getNoteTreeFlattened(this.item, {
+      keepLink: !!getPref("workspace.outline.keepLinks"),
+    });
     this._outlineContainer.contentWindow?.postMessage(
       {
         type: "setMindMapData",
-        nodes: this._addon.api.note.getNoteTreeFlattened(this.item, {
-          keepLink: !!getPref("workspace.outline.keepLinks"),
-        }),
+        nodes,
         expandLevel: getPref("workspace.outline.expandLevel"),
       },
       "*",
@@ -316,13 +317,13 @@ export class OutlinePane extends PluginCEBase {
       }
       case "moveNode": {
         if (!this.item) return;
-        const tree = this._addon.api.note.getNoteTree(this.item);
-        const fromNode = this._addon.api.note.getNoteTreeNodeById(
+        const tree = await this._addon.api.note.getNoteTree(this.item);
+        const fromNode = await this._addon.api.note.getNoteTreeNodeById(
           this.item,
           ev.data.fromID,
           tree,
         );
-        const toNode = this._addon.api.note.getNoteTreeNodeById(
+        const toNode = await this._addon.api.note.getNoteTreeNodeById(
           this.item,
           ev.data.toID,
           tree,

src/extras/parsingWorker.ts

@@ -0,0 +1,114 @@
+import { MessageHelper } from "zotero-plugin-toolkit";
+
+export { handlers };
+
+function parseHTMLLines(html: string): string[] {
+  const randomString: string = `${Math.random()}`;
+  console.time(`parseHTMLLines-${randomString}`);
+
+  // Remove container with one of the attrs named data-schema-version if exists
+  if (html.includes("data-schema-version")) {
+    html = html.replace(/<div[^>]*data-schema-version[^>]*>/, "");
+    html = html.replace(/<\/div>/, "");
+  }
+  const noteLines = html.split("\n").filter((e) => e);
+
+  // A cache for temporarily stored lines
+  let previousLineCache = [];
+  let nextLineCache = [];
+
+  const forceInline = ["table", "blockquote", "pre", "ol", "ul"];
+  const selfInline: string[] = [];
+  const forceInlineStack = [];
+  let forceInlineFlag = false;
+  let selfInlineFlag = false;
+
+  const parsedLines = [];
+  for (const line of noteLines) {
+    // restore self inline flag
+    selfInlineFlag = false;
+
+    // For force inline tags, set flag to append lines to current line
+    for (const tag of forceInline) {
+      const startReg = `<${tag}`;
+      const isStart = line.includes(startReg);
+      const endReg = `</${tag}>`;
+      const isEnd = line.includes(endReg);
+      if (isStart && !isEnd) {
+        forceInlineStack.push(tag);
+        // console.log("push", tag, line, forceInlineStack);
+        forceInlineFlag = true;
+        break;
+      }
+      if (isEnd && !isStart) {
+        forceInlineStack.pop();
+        // console.log("pop", tag, line, forceInlineStack);
+        // Exit force inline mode if the stack is empty
+        if (forceInlineStack.length === 0) {
+          forceInlineFlag = false;
+        }
+        break;
+      }
+    }
+
+    if (forceInlineFlag) {
+      nextLineCache.push(line);
+    } else {
+      // For self inline tags, cache start as previous line and end as next line
+      for (const tag of selfInline) {
+        const isStart = line.includes(`<${tag}`);
+        const isEnd = line.includes(`</${tag}>`);
+        if (isStart && !isEnd) {
+          selfInlineFlag = true;
+          nextLineCache.push(line);
+          break;
+        }
+        if (!isStart && isEnd) {
+          selfInlineFlag = true;
+          previousLineCache.push(line);
+          break;
+        }
+      }
+
+      if (!selfInlineFlag) {
+        // Append cache to previous line
+        if (previousLineCache.length) {
+          parsedLines[parsedLines.length - 1] += `\n${previousLineCache.join(
+            "\n",
+          )}`;
+          previousLineCache = [];
+        }
+        let nextLine = "";
+        // Append cache to next line
+        if (nextLineCache.length) {
+          nextLine = nextLineCache.join("\n");
+          nextLineCache = [];
+        }
+        if (nextLine) {
+          nextLine += "\n";
+        }
+        nextLine += `${line}`;
+        parsedLines.push(nextLine);
+      }
+    }
+  }
+  console.timeEnd(`parseHTMLLines-${randomString}`);
+
+  return parsedLines;
+}
+
+const funcs = {
+  parseHTMLLines,
+};
+
+const handlers = MessageHelper.wrapHandlers(funcs);
+
+const messageServer = new MessageHelper({
+  canBeDestroyed: true,
+  dev: true,
+  name: "parsingWorker",
+  target: self,
+  handlers,
+});
+
+messageServer.start();

src/hooks.ts

@@ -46,6 +46,7 @@ import { closeRelationWorker } from "./utils/relation";
 import { registerNoteLinkSection } from "./modules/workspace/link";
 import { showUserGuide } from "./modules/userGuide";
 import { refreshTemplatesInNote } from "./modules/template/refresh";
+import { closeParsingServer } from "./utils/parsing";
 
 async function onStartup() {
   await Promise.all([
@@ -109,6 +110,7 @@ async function onMainWindowUnload(win: Window): Promise<void> {
 
 function onShutdown(): void {
   closeRelationWorker();
+  closeParsingServer();
   ztoolkit.unregisterAll();
   // Remove addon object
   addon.data.alive = false;

src/modules/editor/toolbar.ts

@@ -71,7 +71,7 @@ async function getMenuData(editor: Zotero.EditorInstance) {
   const noteItem = editor._item;
 
   const currentLine = getLineAtCursor(editor);
-  const currentSection = getSectionAtCursor(editor) || "";
+  const currentSection = (await getSectionAtCursor(editor)) || "";
   const settingsMenuData: PopupData[] = [
     {
       id: makeId("settings-openAsTab"),

src/modules/export/api.ts

@@ -217,7 +217,7 @@ async function embedLinkedNotes(noteItem: Zotero.Item): Promise<string> {
   const globalCitationData = getNoteCitationData(noteItem as Zotero.Item);
 
   const newLines: string[] = [];
-  const noteLines = getLinesInNote(noteItem);
+  const noteLines = await getLinesInNote(noteItem);
   for (const i in noteLines) {
     newLines.push(noteLines[i]);
     const doc = parser.parseFromString(noteLines[i], "text/html");

src/modules/export/freemind.ts

@@ -15,7 +15,7 @@ async function note2mm(
   noteItem: Zotero.Item,
   options: { withContent?: boolean } = { withContent: true },
 ) {
-  const root = getNoteTree(noteItem, false);
+  const root = await getNoteTree(noteItem, false);
   const textNodeForEach = (e: Node, callbackfn: (e: any) => void) => {
     if (e.nodeType === Zotero.getMainWindow().document.TEXT_NODE) {
       callbackfn(e);
@@ -32,7 +32,7 @@ async function note2mm(
     textNodeForEach(doc.body, (e: Text) => {
       e.data = htmlEscape(doc, e.data);
     });
-    lines = parseHTMLLines(doc.body.innerHTML).map((line) =>
+    lines = (await parseHTMLLines(doc.body.innerHTML)).map((line) =>
       htmlUnescape(line),
     );
   }

src/modules/template/refresh.ts

@@ -4,7 +4,7 @@ import { htmlUnescape } from "../../utils/str";
 export { refreshTemplatesInNote };
 
 async function refreshTemplatesInNote(editor: Zotero.EditorInstance) {
-  const lines = addon.api.note.getLinesInNote(editor._item);
+  const lines = await addon.api.note.getLinesInNote(editor._item);
   let startIndex = -1;
   const matchedIndexPairs: { from: number; to: number }[] = [];
 

src/utils/convert.ts

@@ -212,13 +212,15 @@ async function link2html(
     let lineIndex = linkParams.lineIndex;
 
     if (typeof linkParams.sectionName === "string") {
-      const sectionTree = addon.api.note.getNoteTreeFlattened(item);
+      const sectionTree = await addon.api.note.getNoteTreeFlattened(item);
       const sectionNode = sectionTree.find(
         (node) => node.model.name.trim() === linkParams.sectionName!.trim(),
       );
       lineIndex = sectionNode?.model.lineIndex;
     }
-    html = addon.api.note.getLinesInNote(item).slice(lineIndex).join("\n");
+    html = (await addon.api.note.getLinesInNote(item))
+      .slice(lineIndex)
+      .join("\n");
   } else {
     html = addon.api.sync.getNoteStatus(linkParams.noteItem.id)?.content || "";
   }

src/utils/editor.ts

@@ -136,9 +136,12 @@ function scroll(editor: Zotero.EditorInstance, lineIndex: number) {
   core.view.dom.parentElement?.scrollTo(0, offset);
 }
 
-function scrollToSection(editor: Zotero.EditorInstance, sectionName: string) {
+async function scrollToSection(
+  editor: Zotero.EditorInstance,
+  sectionName: string,
+) {
   const item = editor._item;
-  const sectionTree = getNoteTreeFlattened(item);
+  const sectionTree = await getNoteTreeFlattened(item);
   const sectionNode = sectionTree.find(
     (node) => node.model.name.trim() === sectionName.trim(),
   );
@@ -199,11 +202,13 @@ function getLineAtCursor(editor: Zotero.EditorInstance) {
   return i;
 }
 
-function getSectionAtCursor(editor: Zotero.EditorInstance): string | undefined {
+async function getSectionAtCursor(
+  editor: Zotero.EditorInstance,
+): Promise<string | undefined> {
   const lineIndex = getLineAtCursor(editor);
   if (lineIndex < 0) return undefined;
   const item = editor._item;
-  const sectionTree = getNoteTreeFlattened(item);
+  const sectionTree = await getNoteTreeFlattened(item);
   let sectionNode;
   for (let i = 0; i < sectionTree.length; i++) {
     if (