From 68b0c95acedb017c96fe6eed75d23deb5f7e05f1 Mon Sep 17 00:00:00 2001 From: Yufei Huang Date: Tue, 3 Oct 2023 21:37:05 +0800 Subject: [PATCH] feat: enhance lunr search --- templates/build.js | 3 + templates/modern/layout/_master.tmpl | 1 - templates/modern/src/options.d.ts | 4 + templates/modern/src/search-worker.ts | 105 +++++++++++--------------- templates/package-lock.json | 24 ++++++ templates/package.json | 2 + 6 files changed, 77 insertions(+), 62 deletions(-) diff --git a/templates/build.js b/templates/build.js index 48def0354c6..63606e55574 100644 --- a/templates/build.js +++ b/templates/build.js @@ -52,6 +52,9 @@ async function buildModernTemplate() { 'modern/src/docfx.ts', 'modern/src/search-worker.ts', ], + external: [ + './main.js' + ], plugins: [ sassPlugin() ], diff --git a/templates/modern/layout/_master.tmpl b/templates/modern/layout/_master.tmpl index 1b1cc0830a2..6b8bdff7fb9 100644 --- a/templates/modern/layout/_master.tmpl +++ b/templates/modern/layout/_master.tmpl @@ -2,7 +2,6 @@ {{!include(/^public/.*/)}} {{!include(favicon.ico)}} {{!include(logo.svg)}} -{{!include(search-stopwords.json)}} diff --git a/templates/modern/src/options.d.ts b/templates/modern/src/options.d.ts index 2e93b195108..a088e294b84 100644 --- a/templates/modern/src/options.d.ts +++ b/templates/modern/src/options.d.ts @@ -5,6 +5,7 @@ import BootstrapIcons from 'bootstrap-icons/font/bootstrap-icons.json' import { HLJSApi } from 'highlight.js' import { AnchorJSOptions } from 'anchor-js' import { MermaidConfig } from 'mermaid' +import lunr from 'lunr' export type Theme = 'light' | 'dark' | 'auto' @@ -37,4 +38,7 @@ export type DocfxOptions = { /** Configures [hightlight.js](https://highlightjs.org/) */ configureHljs?: (hljs: HLJSApi) => void, + + /** Configures [lunr](https://lunrjs.com/docs/index.html) */ + configureLunr?: (lunr: lunr.Builder) => void, } diff --git a/templates/modern/src/search-worker.ts b/templates/modern/src/search-worker.ts index 381d586d026..7d48dc60732 100644 --- a/templates/modern/src/search-worker.ts +++ b/templates/modern/src/search-worker.ts @@ -2,81 +2,64 @@ // The .NET Foundation licenses this file to you under the MIT license. import lunr from 'lunr' +import { get, set, createStore } from 'idb-keyval' +import { DocfxOptions } from './options' -let lunrIndex - -let stopWords = null -let searchData = {} - -lunr.tokenizer.separator = /[\s\-.()]+/ - -const stopWordsRequest = new XMLHttpRequest() -stopWordsRequest.open('GET', '../search-stopwords.json') -stopWordsRequest.onload = function() { - if (this.status !== 200) { - return - } - stopWords = JSON.parse(this.responseText) - buildIndex() +type SearchHit = { + href: string + title: string + keywords: string } -stopWordsRequest.send() - -const searchDataRequest = new XMLHttpRequest() - -searchDataRequest.open('GET', '../index.json') -searchDataRequest.onload = function() { - if (this.status !== 200) { - return - } - searchData = JSON.parse(this.responseText) - buildIndex() +let search: (q: string) => SearchHit[] +async function loadIndex() { + const { index, data } = await loadIndexCore() + search = q => index.search(q).map(({ ref }) => data[ref]) postMessage({ e: 'index-ready' }) } -searchDataRequest.send() -onmessage = function(oEvent) { - const q = oEvent.data.q - const results = [] - if (lunrIndex) { - const hits = lunrIndex.search(q) - hits.forEach(function(hit) { - const item = searchData[hit.ref] - results.push({ href: item.href, title: item.title, keywords: item.keywords }) - }) +async function loadIndexCore() { + const res = await fetch('../index.json') + const etag = res.headers.get('etag') + const data = await res.json() as { [key: string]: SearchHit } + const cache = createStore('docfx', 'lunr') + + const main = await import('./main.js') + const docfx = main.default as DocfxOptions + + if (etag) { + const value = JSON.parse(await get('index', cache) || '{}') + if (value && value.etag === etag) { + return { index: lunr.Index.load(value), data } + } } - postMessage({ e: 'query-ready', q, d: results }) -} -function buildIndex() { - if (stopWords !== null && !isEmpty(searchData)) { - lunrIndex = lunr(function() { - this.pipeline.remove(lunr.stopWordFilter) - this.ref('href') - this.field('title', { boost: 50 }) - this.field('keywords', { boost: 20 }) + const index = lunr(function() { + this.pipeline.remove(lunr.stopWordFilter) + this.ref('href') + this.field('title', { boost: 50 }) + this.field('keywords', { boost: 20 }) - for (const prop in searchData) { - if (Object.prototype.hasOwnProperty.call(searchData, prop)) { - this.add(searchData[prop]) - } - } + lunr.tokenizer.separator = /[\s\-.()]+/ + docfx.configureLunr?.(this) - const docfxStopWordFilter = lunr.generateStopWordFilter(stopWords) - lunr.Pipeline.registerFunction(docfxStopWordFilter, 'docfxStopWordFilter') - this.pipeline.add(docfxStopWordFilter) - this.searchPipeline.add(docfxStopWordFilter) - }) + for (const key in data) { + this.add(data[key]) + } + }) + + if (etag) { + await set('index', JSON.stringify(Object.assign(index.toJSON(), { etag })), cache) } + + return { index, data } } -function isEmpty(obj) { - if (!obj) return true +loadIndex().catch(console.error) - for (const prop in obj) { - if (Object.prototype.hasOwnProperty.call(obj, prop)) { return false } +onmessage = function(e) { + if (search) { + postMessage({ e: 'query-ready', d: search(e.data.q) }) } - - return true } diff --git a/templates/package-lock.json b/templates/package-lock.json index 093007b384e..b71958a4b8f 100644 --- a/templates/package-lock.json +++ b/templates/package-lock.json @@ -21,6 +21,7 @@ "bootstrap": "^5.3.2", "bootstrap-icons": "^1.11.1", "highlight.js": "^11.8.0", + "idb-keyval": "^6.2.1", "jquery": "3.7.0", "lit-html": "^2.8.0", "lunr": "2.3.9", @@ -29,6 +30,7 @@ }, "devDependencies": { "@types/jest": "^29.5.5", + "@types/lunr": "^2.3.5", "@typescript-eslint/eslint-plugin": "^6.7.4", "@typescript-eslint/parser": "^6.7.4", "browser-sync": "^2.29.3", @@ -2034,6 +2036,12 @@ "dev": true, "peer": true }, + "node_modules/@types/lunr": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@types/lunr/-/lunr-2.3.5.tgz", + "integrity": "sha512-C4xYh7A4FRKg70AWJCe27oJYVPhUlEY5MQ4dKbRR7G6Xsb2HiqO672yWHRvWxl8/h7IuISvwDjv88ECHUEsV2A==", + "dev": true + }, "node_modules/@types/mdast": { "version": "3.0.11", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.11.tgz", @@ -5583,6 +5591,11 @@ "node": ">=0.10.0" } }, + "node_modules/idb-keyval": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.1.tgz", + "integrity": "sha512-8Sb3veuYCyrZL+VBt9LJfZjLUPWVvqn8tG28VqYNFCo43KHcKuq+b4EiXGeuaLAQWL2YmyDgMp2aSpH9JHsEQg==" + }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -11643,6 +11656,12 @@ "dev": true, "peer": true }, + "@types/lunr": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@types/lunr/-/lunr-2.3.5.tgz", + "integrity": "sha512-C4xYh7A4FRKg70AWJCe27oJYVPhUlEY5MQ4dKbRR7G6Xsb2HiqO672yWHRvWxl8/h7IuISvwDjv88ECHUEsV2A==", + "dev": true + }, "@types/mdast": { "version": "3.0.11", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.11.tgz", @@ -14235,6 +14254,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "idb-keyval": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.1.tgz", + "integrity": "sha512-8Sb3veuYCyrZL+VBt9LJfZjLUPWVvqn8tG28VqYNFCo43KHcKuq+b4EiXGeuaLAQWL2YmyDgMp2aSpH9JHsEQg==" + }, "ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", diff --git a/templates/package.json b/templates/package.json index d60c5235245..60203133338 100644 --- a/templates/package.json +++ b/templates/package.json @@ -30,6 +30,7 @@ "bootstrap": "^5.3.2", "bootstrap-icons": "^1.11.1", "highlight.js": "^11.8.0", + "idb-keyval": "^6.2.1", "jquery": "3.7.0", "lit-html": "^2.8.0", "lunr": "2.3.9", @@ -38,6 +39,7 @@ }, "devDependencies": { "@types/jest": "^29.5.5", + "@types/lunr": "^2.3.5", "@typescript-eslint/eslint-plugin": "^6.7.4", "@typescript-eslint/parser": "^6.7.4", "browser-sync": "^2.29.3",