From da68a449d2ef9ab5f8f17a3e68d19bf30135d10e Mon Sep 17 00:00:00 2001 From: windingwind <33902321+windingwind@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:37:32 +0200 Subject: [PATCH] add: parsing worker fix: #1166 --- package-lock.json | 8 +- package.json | 2 +- src/addon.ts | 6 + src/elements/linkCreator/outlinePicker.ts | 4 +- src/elements/workspace/outlinePane.ts | 13 ++- src/extras/parsingWorker.ts | 114 ++++++++++++++++++ src/hooks.ts | 2 + src/modules/editor/toolbar.ts | 2 +- src/modules/export/api.ts | 2 +- src/modules/export/freemind.ts | 4 +- src/modules/template/refresh.ts | 2 +- src/utils/convert.ts | 6 +- src/utils/editor.ts | 13 ++- src/utils/note.ts | 134 ++++------------------ src/utils/parsing.ts | 33 ++++++ src/utils/relation.ts | 2 +- 16 files changed, 212 insertions(+), 135 deletions(-) create mode 100644 src/extras/parsingWorker.ts create mode 100644 src/utils/parsing.ts diff --git a/package-lock.json b/package-lock.json index f62a19e8..a419034b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,7 @@ "unist-util-visit": "^5.0.0", "unist-util-visit-parents": "^6.0.1", "yamljs": "^0.3.0", - "zotero-plugin-toolkit": "^4.0.6" + "zotero-plugin-toolkit": "^4.0.7" }, "devDependencies": { "@esbuild-plugins/node-globals-polyfill": "^0.2.3", @@ -13588,9 +13588,9 @@ } }, "node_modules/zotero-plugin-toolkit": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/zotero-plugin-toolkit/-/zotero-plugin-toolkit-4.0.6.tgz", - "integrity": "sha512-juxIrSrUYTxk+efQJAH7OpfQHSboErMd0Ygu2eZs/xKA1177XlCYhe0sdxlTxI8Y/4UGtRwLicThUddaOfArMQ==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/zotero-plugin-toolkit/-/zotero-plugin-toolkit-4.0.7.tgz", + "integrity": "sha512-9bAXXJTgH5SkyauGft50EerudZYGYzN/T3kPgi7udB8lScVyO+HkYERxJqnFuvk2gHzV+stqeaKaX9a0ODTxEQ==", "dependencies": { "zotero-types": "^2.2.0" }, diff --git a/package.json b/package.json index 5b2ecea8..03100e15 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "unist-util-visit": "^5.0.0", "unist-util-visit-parents": "^6.0.1", "yamljs": "^0.3.0", - "zotero-plugin-toolkit": "^4.0.6" + "zotero-plugin-toolkit": "^4.0.7" }, "devDependencies": { "@esbuild-plugins/node-globals-polyfill": "^0.2.3", diff --git a/src/addon.ts b/src/addon.ts index 069deca4..7b3bebda 100644 --- a/src/addon.ts +++ b/src/addon.ts @@ -5,6 +5,8 @@ import { SyncDataType } from "./modules/sync/managerWindow"; import hooks from "./hooks"; import api from "./api"; import { createZToolkit } from "./utils/ztoolkit"; +import { MessageHelper } from "zotero-plugin-toolkit/dist/helpers/message"; +import type { handlers } from "./extras/parsingWorker"; class Addon { public data: { @@ -70,6 +72,9 @@ class Addon { relation: { worker?: Worker; }; + parsing: { + server?: MessageHelper; + }; imageCache: Record; hint: { silent: boolean; @@ -117,6 +122,7 @@ class Addon { }, }, relation: {}, + parsing: {}, imageCache: {}, hint: { silent: false, diff --git a/src/elements/linkCreator/outlinePicker.ts b/src/elements/linkCreator/outlinePicker.ts index 6e19726d..38f17395 100644 --- a/src/elements/linkCreator/outlinePicker.ts +++ b/src/elements/linkCreator/outlinePicker.ts @@ -131,7 +131,9 @@ export class OutlinePicker extends PluginCEBase { if (!this.item) { return; } - this.noteOutline = this._addon.api.note.getNoteTreeFlattened(this.item); + this.noteOutline = await this._addon.api.note.getNoteTreeFlattened( + this.item, + ); // Fake a cursor position if (typeof this.lineIndex === "number") { // @ts-ignore - formatValues is not in the types diff --git a/src/elements/workspace/outlinePane.ts b/src/elements/workspace/outlinePane.ts index 4f3b1f98..82973637 100644 --- a/src/elements/workspace/outlinePane.ts +++ b/src/elements/workspace/outlinePane.ts @@ -211,12 +211,13 @@ export class OutlinePane extends PluginCEBase { "message", this.messageHandler, ); + const nodes = await this._addon.api.note.getNoteTreeFlattened(this.item, { + keepLink: !!getPref("workspace.outline.keepLinks"), + }); this._outlineContainer.contentWindow?.postMessage( { type: "setMindMapData", - nodes: this._addon.api.note.getNoteTreeFlattened(this.item, { - keepLink: !!getPref("workspace.outline.keepLinks"), - }), + nodes, expandLevel: getPref("workspace.outline.expandLevel"), }, "*", @@ -316,13 +317,13 @@ export class OutlinePane extends PluginCEBase { } case "moveNode": { if (!this.item) return; - const tree = this._addon.api.note.getNoteTree(this.item); - const fromNode = this._addon.api.note.getNoteTreeNodeById( + const tree = await this._addon.api.note.getNoteTree(this.item); + const fromNode = await this._addon.api.note.getNoteTreeNodeById( this.item, ev.data.fromID, tree, ); - const toNode = this._addon.api.note.getNoteTreeNodeById( + const toNode = await this._addon.api.note.getNoteTreeNodeById( this.item, ev.data.toID, tree, diff --git a/src/extras/parsingWorker.ts b/src/extras/parsingWorker.ts new file mode 100644 index 00000000..3b5cd5ee --- /dev/null +++ b/src/extras/parsingWorker.ts @@ -0,0 +1,114 @@ +import { MessageHelper } from "zotero-plugin-toolkit"; + +export { handlers }; + +function parseHTMLLines(html: string): string[] { + const randomString: string = `${Math.random()}`; + console.time(`parseHTMLLines-${randomString}`); + + // Remove container with one of the attrs named data-schema-version if exists + if (html.includes("data-schema-version")) { + html = html.replace(/]*data-schema-version[^>]*>/, ""); + html = html.replace(/<\/div>/, ""); + } + const noteLines = html.split("\n").filter((e) => e); + + // A cache for temporarily stored lines + let previousLineCache = []; + let nextLineCache = []; + + const forceInline = ["table", "blockquote", "pre", "ol", "ul"]; + const selfInline: string[] = []; + const forceInlineStack = []; + let forceInlineFlag = false; + let selfInlineFlag = false; + + const parsedLines = []; + for (const line of noteLines) { + // restore self inline flag + selfInlineFlag = false; + + // For force inline tags, set flag to append lines to current line + for (const tag of forceInline) { + const startReg = `<${tag}`; + const isStart = line.includes(startReg); + const endReg = ``; + const isEnd = line.includes(endReg); + if (isStart && !isEnd) { + forceInlineStack.push(tag); + // console.log("push", tag, line, forceInlineStack); + forceInlineFlag = true; + break; + } + if (isEnd && !isStart) { + forceInlineStack.pop(); + // console.log("pop", tag, line, forceInlineStack); + // Exit force inline mode if the stack is empty + if (forceInlineStack.length === 0) { + forceInlineFlag = false; + } + break; + } + } + + if (forceInlineFlag) { + nextLineCache.push(line); + } else { + // For self inline tags, cache start as previous line and end as next line + for (const tag of selfInline) { + const isStart = line.includes(`<${tag}`); + const isEnd = line.includes(``); + if (isStart && !isEnd) { + selfInlineFlag = true; + nextLineCache.push(line); + break; + } + if (!isStart && isEnd) { + selfInlineFlag = true; + previousLineCache.push(line); + break; + } + } + + if (!selfInlineFlag) { + // Append cache to previous line + if (previousLineCache.length) { + parsedLines[parsedLines.length - 1] += `\n${previousLineCache.join( + "\n", + )}`; + previousLineCache = []; + } + let nextLine = ""; + // Append cache to next line + if (nextLineCache.length) { + nextLine = nextLineCache.join("\n"); + nextLineCache = []; + } + if (nextLine) { + nextLine += "\n"; + } + nextLine += `${line}`; + parsedLines.push(nextLine); + } + } + } + console.timeEnd(`parseHTMLLines-${randomString}`); + + return parsedLines; +} + +const funcs = { + parseHTMLLines, +}; + +const handlers = MessageHelper.wrapHandlers(funcs); + +const messageServer = new MessageHelper({ + canBeDestroyed: true, + dev: true, + name: "parsingWorker", + target: self, + handlers, +}); + +messageServer.start(); diff --git a/src/hooks.ts b/src/hooks.ts index 88c6aaf9..5a7c633b 100644 --- a/src/hooks.ts +++ b/src/hooks.ts @@ -46,6 +46,7 @@ import { closeRelationWorker } from "./utils/relation"; import { registerNoteLinkSection } from "./modules/workspace/link"; import { showUserGuide } from "./modules/userGuide"; import { refreshTemplatesInNote } from "./modules/template/refresh"; +import { closeParsingServer } from "./utils/parsing"; async function onStartup() { await Promise.all([ @@ -109,6 +110,7 @@ async function onMainWindowUnload(win: Window): Promise { function onShutdown(): void { closeRelationWorker(); + closeParsingServer(); ztoolkit.unregisterAll(); // Remove addon object addon.data.alive = false; diff --git a/src/modules/editor/toolbar.ts b/src/modules/editor/toolbar.ts index 193d00fe..2d899a0f 100644 --- a/src/modules/editor/toolbar.ts +++ b/src/modules/editor/toolbar.ts @@ -71,7 +71,7 @@ async function getMenuData(editor: Zotero.EditorInstance) { const noteItem = editor._item; const currentLine = getLineAtCursor(editor); - const currentSection = getSectionAtCursor(editor) || ""; + const currentSection = (await getSectionAtCursor(editor)) || ""; const settingsMenuData: PopupData[] = [ { id: makeId("settings-openAsTab"), diff --git a/src/modules/export/api.ts b/src/modules/export/api.ts index 91c4c183..534edd7a 100644 --- a/src/modules/export/api.ts +++ b/src/modules/export/api.ts @@ -217,7 +217,7 @@ async function embedLinkedNotes(noteItem: Zotero.Item): Promise { const globalCitationData = getNoteCitationData(noteItem as Zotero.Item); const newLines: string[] = []; - const noteLines = getLinesInNote(noteItem); + const noteLines = await getLinesInNote(noteItem); for (const i in noteLines) { newLines.push(noteLines[i]); const doc = parser.parseFromString(noteLines[i], "text/html"); diff --git a/src/modules/export/freemind.ts b/src/modules/export/freemind.ts index 1950fdd4..41460c58 100644 --- a/src/modules/export/freemind.ts +++ b/src/modules/export/freemind.ts @@ -15,7 +15,7 @@ async function note2mm( noteItem: Zotero.Item, options: { withContent?: boolean } = { withContent: true }, ) { - const root = getNoteTree(noteItem, false); + const root = await getNoteTree(noteItem, false); const textNodeForEach = (e: Node, callbackfn: (e: any) => void) => { if (e.nodeType === Zotero.getMainWindow().document.TEXT_NODE) { callbackfn(e); @@ -32,7 +32,7 @@ async function note2mm( textNodeForEach(doc.body, (e: Text) => { e.data = htmlEscape(doc, e.data); }); - lines = parseHTMLLines(doc.body.innerHTML).map((line) => + lines = (await parseHTMLLines(doc.body.innerHTML)).map((line) => htmlUnescape(line), ); } diff --git a/src/modules/template/refresh.ts b/src/modules/template/refresh.ts index ca7484ec..373de681 100644 --- a/src/modules/template/refresh.ts +++ b/src/modules/template/refresh.ts @@ -4,7 +4,7 @@ import { htmlUnescape } from "../../utils/str"; export { refreshTemplatesInNote }; async function refreshTemplatesInNote(editor: Zotero.EditorInstance) { - const lines = addon.api.note.getLinesInNote(editor._item); + const lines = await addon.api.note.getLinesInNote(editor._item); let startIndex = -1; const matchedIndexPairs: { from: number; to: number }[] = []; diff --git a/src/utils/convert.ts b/src/utils/convert.ts index c5a87310..61d92e91 100644 --- a/src/utils/convert.ts +++ b/src/utils/convert.ts @@ -212,13 +212,15 @@ async function link2html( let lineIndex = linkParams.lineIndex; if (typeof linkParams.sectionName === "string") { - const sectionTree = addon.api.note.getNoteTreeFlattened(item); + const sectionTree = await addon.api.note.getNoteTreeFlattened(item); const sectionNode = sectionTree.find( (node) => node.model.name.trim() === linkParams.sectionName!.trim(), ); lineIndex = sectionNode?.model.lineIndex; } - html = addon.api.note.getLinesInNote(item).slice(lineIndex).join("\n"); + html = (await addon.api.note.getLinesInNote(item)) + .slice(lineIndex) + .join("\n"); } else { html = addon.api.sync.getNoteStatus(linkParams.noteItem.id)?.content || ""; } diff --git a/src/utils/editor.ts b/src/utils/editor.ts index 8257c187..11494906 100644 --- a/src/utils/editor.ts +++ b/src/utils/editor.ts @@ -136,9 +136,12 @@ function scroll(editor: Zotero.EditorInstance, lineIndex: number) { core.view.dom.parentElement?.scrollTo(0, offset); } -function scrollToSection(editor: Zotero.EditorInstance, sectionName: string) { +async function scrollToSection( + editor: Zotero.EditorInstance, + sectionName: string, +) { const item = editor._item; - const sectionTree = getNoteTreeFlattened(item); + const sectionTree = await getNoteTreeFlattened(item); const sectionNode = sectionTree.find( (node) => node.model.name.trim() === sectionName.trim(), ); @@ -199,11 +202,13 @@ function getLineAtCursor(editor: Zotero.EditorInstance) { return i; } -function getSectionAtCursor(editor: Zotero.EditorInstance): string | undefined { +async function getSectionAtCursor( + editor: Zotero.EditorInstance, +): Promise { const lineIndex = getLineAtCursor(editor); if (lineIndex < 0) return undefined; const item = editor._item; - const sectionTree = getNoteTreeFlattened(item); + const sectionTree = await getNoteTreeFlattened(item); let sectionNode; for (let i = 0; i < sectionTree.length; i++) { if ( diff --git a/src/utils/note.ts b/src/utils/note.ts index 4ca128e2..56995e3e 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -2,8 +2,8 @@ import TreeModel = require("tree-model"); import katex = require("katex"); import { getEditorInstance, getPositionAtLine, insert } from "./editor"; import { formatPath, getItemDataURL } from "./str"; -import { showHint } from "./hint"; import { config } from "../../package.json"; +import { getParsingServer } from "./parsing"; export { renderNoteHTML, @@ -18,111 +18,17 @@ export { importImageToNote, }; -function parseHTMLLines(html: string): string[] { - // Remove container with one of the attrs named data-schema-version if exists - if (html.includes("data-schema-version")) { - html = html.replace(/]*data-schema-version[^>]*>/, ""); - html = html.replace(/<\/div>/, ""); - } - const noteLines = html.split("\n").filter((e) => e); - - // A cache for temporarily stored lines - let previousLineCache = []; - let nextLineCache = []; - - const forceInline = ["table", "blockquote", "pre", "ol", "ul"]; - const selfInline: string[] = []; - const forceInlineStack = []; - let forceInlineFlag = false; - let selfInlineFlag = false; - - const parsedLines = []; - for (const line of noteLines) { - // restore self inline flag - selfInlineFlag = false; - - // For force inline tags, set flag to append lines to current line - for (const tag of forceInline) { - const startReg = `<${tag}`; - const isStart = line.includes(startReg); - const endReg = ``; - const isEnd = line.includes(endReg); - if (isStart && !isEnd) { - forceInlineStack.push(tag); - ztoolkit.log("push", tag, line, forceInlineStack); - forceInlineFlag = true; - break; - } - if (isEnd && !isStart) { - forceInlineStack.pop(); - ztoolkit.log("pop", tag, line, forceInlineStack); - // Exit force inline mode if the stack is empty - if (forceInlineStack.length === 0) { - forceInlineFlag = false; - } - break; - } - } - - if (forceInlineFlag) { - nextLineCache.push(line); - } else { - // For self inline tags, cache start as previous line and end as next line - for (const tag of selfInline) { - const isStart = line.includes(`<${tag}`); - const isEnd = line.includes(``); - if (isStart && !isEnd) { - selfInlineFlag = true; - nextLineCache.push(line); - break; - } - if (!isStart && isEnd) { - selfInlineFlag = true; - previousLineCache.push(line); - break; - } - } - - if (!selfInlineFlag) { - // Append cache to previous line - if (previousLineCache.length) { - parsedLines[parsedLines.length - 1] += `\n${previousLineCache.join( - "\n", - )}`; - previousLineCache = []; - } - let nextLine = ""; - // Append cache to next line - if (nextLineCache.length) { - nextLine = nextLineCache.join("\n"); - nextLineCache = []; - } - if (nextLine) { - nextLine += "\n"; - } - nextLine += `${line}`; - parsedLines.push(nextLine); - } - } - } - return parsedLines; +async function parseHTMLLines(html: string) { + const server = await getParsingServer(); + return await server.exec("parseHTMLLines", [html]); } -function getLinesInNote(note: Zotero.Item): string[]; - async function getLinesInNote( - note: Zotero.Item, - options: { - convertToHTML?: true; - }, -): Promise; - -function getLinesInNote( note: Zotero.Item, options?: { convertToHTML?: boolean; }, -): string[] | Promise { +): Promise { if (!note) { return []; } @@ -170,7 +76,7 @@ async function addLineToNote( if (!note || !html) { return; } - const noteLines = getLinesInNote(note); + const noteLines = await getLinesInNote(note); if (lineIndex < 0 || lineIndex >= noteLines.length) { lineIndex = noteLines.length; } @@ -292,11 +198,14 @@ async function renderNoteHTML( return doc.body.innerHTML; } -function getNoteTree( +async function getNoteTree( note: Zotero.Item, parseLink: boolean = true, -): TreeModel.Node { - const noteLines = getLinesInNote(note); +): Promise> { + const timeLabel = `getNoteTree-${note.id}-${Math.random()}`; + Zotero.getMainWindow().console.time(timeLabel); + + const noteLines = await getLinesInNote(note); const parser = new DOMParser(); const tree = new TreeModel(); const root = tree.parse({ @@ -353,6 +262,8 @@ function getNoteTree( if (node.model.endIndex > parseInt(i) - 1) { node.model.endIndex = parseInt(i) - 1; } + Zotero.getMainWindow().console.timeEnd(timeLabel); + return true; }); previousNode.model.endIndex = parseInt(i) - 1; @@ -360,21 +271,22 @@ function getNoteTree( lastNode = currentNode; } } + Zotero.getMainWindow().console.timeEnd(timeLabel); return root; } -function getNoteTreeFlattened( +async function getNoteTreeFlattened( note: Zotero.Item, options: { keepRoot?: boolean; keepLink?: boolean; customFilter?: (node: TreeModel.Node) => boolean; } = { keepRoot: false, keepLink: false }, -): TreeModel.Node[] { +): Promise[]> { if (!note) { return []; } - return getNoteTree(note).all( + return (await getNoteTree(note)).all( (node) => (options.keepRoot || node.model.lineIndex >= 0) && (options.keepLink || node.model.level <= 6) && @@ -382,23 +294,23 @@ function getNoteTreeFlattened( ); } -function getNoteTreeNodeById( +async function getNoteTreeNodeById( note: Zotero.Item, id: number, root: TreeModel.Node | undefined = undefined, ) { - root = root || getNoteTree(note); + root = root || (await getNoteTree(note)); return root.first(function (node) { return node.model.id === id; }); } -function getNoteTreeNodesByLevel( +async function getNoteTreeNodesByLevel( note: Zotero.Item, level: number, root: TreeModel.Node | undefined = undefined, ) { - root = root || getNoteTree(note); + root = root || (await getNoteTree(note)); return root.all(function (node) { return node.model.level === level; }); @@ -534,7 +446,7 @@ async function importImageToNote( } if (!blob) { - showHint("Failed to import image."); + ztoolkit.log("Failed to import image."); return; } diff --git a/src/utils/parsing.ts b/src/utils/parsing.ts new file mode 100644 index 00000000..fd1931cb --- /dev/null +++ b/src/utils/parsing.ts @@ -0,0 +1,33 @@ +import { MessageHelper } from "zotero-plugin-toolkit"; +import { config } from "../../package.json"; +import type { handlers } from "../extras/parsingWorker"; + +function closeParsingServer() { + if (addon.data.parsing.server) { + addon.data.parsing.server.destroy(); + addon.data.parsing.server = undefined; + } +} + +async function getParsingServer() { + if (addon.data.parsing.server) { + return addon.data.parsing.server; + } + const worker = new ChromeWorker( + `chrome://${config.addonRef}/content/scripts/parsingWorker.js`, + { name: "parsingWorker" }, + ); + const server = new MessageHelper({ + canBeDestroyed: false, + dev: true, + name: "parsingWorkerMain", + target: worker, + handlers: {}, + }); + server.start(); + await server.exec("_ping"); + addon.data.parsing.server = server; + return server; +} + +export { getParsingServer, closeParsingServer }; diff --git a/src/utils/relation.ts b/src/utils/relation.ts index 23e5b47b..6c113788 100644 --- a/src/utils/relation.ts +++ b/src/utils/relation.ts @@ -73,7 +73,7 @@ async function updateNoteLinkRelation(noteID: number) { const affectedNoteIDs = new Set([noteID]); const fromLibID = note.libraryID; const fromKey = note.key; - const lines = addon.api.note.getLinesInNote(note); + const lines = await addon.api.note.getLinesInNote(note); const linkToData: LinkModel[] = []; for (let i = 0; i < lines.length; i++) { const linkMatches = lines[i].match(/href="zotero:\/\/note\/[^"]+"/g);