feat: enhance lunr search for modern template (dotnet#9264)

ErpNetDocs · Jun 28, 2024 · 4e9c472 · 4e9c472
1 parent b93aae3
commit 4e9c472
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 304 deletions.
diff --git a/templates/build.js b/templates/build.js
@@ -52,6 +52,9 @@ async function buildModernTemplate() {
       'modern/src/docfx.ts',
       'modern/src/search-worker.ts',
     ],
+    external: [
+      './main.js'
+    ],
     plugins: [
       sassPlugin()
     ],

diff --git a/templates/modern/layout/_master.tmpl b/templates/modern/layout/_master.tmpl
@@ -2,7 +2,6 @@
 {{!include(/^public/.*/)}}
 {{!include(favicon.ico)}}
 {{!include(logo.svg)}}
-{{!include(search-stopwords.json)}}
 <!DOCTYPE html>
 <html {{#_lang}}lang="{{_lang}}"{{/_lang}}>
   <head>

diff --git a/templates/modern/src/options.d.ts b/templates/modern/src/options.d.ts
@@ -5,6 +5,7 @@ import BootstrapIcons from 'bootstrap-icons/font/bootstrap-icons.json'
 import { HLJSApi } from 'highlight.js'
 import { AnchorJSOptions } from 'anchor-js'
 import { MermaidConfig } from 'mermaid'
+import lunr from 'lunr'
 
 export type Theme = 'light' | 'dark' | 'auto'
 
@@ -37,4 +38,7 @@ export type DocfxOptions = {
 
   /** Configures [hightlight.js](https://highlightjs.org/) */
   configureHljs?: (hljs: HLJSApi) => void,
+
+  /** Configures [lunr](https://lunrjs.com/docs/index.html) */
+  configureLunr?: (lunr: lunr.Builder) => void,
 }
diff --git a/templates/modern/src/search-worker.ts b/templates/modern/src/search-worker.ts
@@ -2,81 +2,64 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 import lunr from 'lunr'
+import { get, set, createStore } from 'idb-keyval'
+import { DocfxOptions } from './options'
 
-let lunrIndex
-
-let stopWords = null
-let searchData = {}
-
-lunr.tokenizer.separator = /[\s\-.()]+/
-
-const stopWordsRequest = new XMLHttpRequest()
-stopWordsRequest.open('GET', '../search-stopwords.json')
-stopWordsRequest.onload = function() {
-  if (this.status !== 200) {
-    return
-  }
-  stopWords = JSON.parse(this.responseText)
-  buildIndex()
+type SearchHit = {
+  href: string
+  title: string
+  keywords: string
 }
-stopWordsRequest.send()
-
-const searchDataRequest = new XMLHttpRequest()
-
-searchDataRequest.open('GET', '../index.json')
-searchDataRequest.onload = function() {
-  if (this.status !== 200) {
-    return
-  }
-  searchData = JSON.parse(this.responseText)
 
-  buildIndex()
+let search: (q: string) => SearchHit[]
 
+async function loadIndex() {
+  const { index, data } = await loadIndexCore()
+  search = q => index.search(q).map(({ ref }) => data[ref])
   postMessage({ e: 'index-ready' })
 }
-searchDataRequest.send()
 
-onmessage = function(oEvent) {
-  const q = oEvent.data.q
-  const results = []
-  if (lunrIndex) {
-    const hits = lunrIndex.search(q)
-    hits.forEach(function(hit) {
-      const item = searchData[hit.ref]
-      results.push({ href: item.href, title: item.title, keywords: item.keywords })
-    })
+async function loadIndexCore() {
+  const res = await fetch('../index.json')
+  const etag = res.headers.get('etag')
+  const data = await res.json() as { [key: string]: SearchHit }
+  const cache = createStore('docfx', 'lunr')
+
+  if (etag) {
+    const value = JSON.parse(await get('index', cache) || '{}')
+    if (value && value.etag === etag) {
+      return { index: lunr.Index.load(value), data }
+    }
   }
-  postMessage({ e: 'query-ready', q, d: results })
-}
 
-function buildIndex() {
-  if (stopWords !== null && !isEmpty(searchData)) {
-    lunrIndex = lunr(function() {
-      this.pipeline.remove(lunr.stopWordFilter)
-      this.ref('href')
-      this.field('title', { boost: 50 })
-      this.field('keywords', { boost: 20 })
+  const main = await import('./main.js')
+  const docfx = main.default as DocfxOptions
+
+  const index = lunr(function() {
+    this.pipeline.remove(lunr.stopWordFilter)
+    this.ref('href')
+    this.field('title', { boost: 50 })
+    this.field('keywords', { boost: 20 })
 
-      for (const prop in searchData) {
-        if (Object.prototype.hasOwnProperty.call(searchData, prop)) {
-          this.add(searchData[prop])
-        }
-      }
+    lunr.tokenizer.separator = /[\s\-.()]+/
+    docfx.configureLunr?.(this)
 
-      const docfxStopWordFilter = lunr.generateStopWordFilter(stopWords)
-      lunr.Pipeline.registerFunction(docfxStopWordFilter, 'docfxStopWordFilter')
-      this.pipeline.add(docfxStopWordFilter)
-      this.searchPipeline.add(docfxStopWordFilter)
-    })
+    for (const key in data) {
+      this.add(data[key])
+    }
+  })
+
+  if (etag) {
+    await set('index', JSON.stringify(Object.assign(index.toJSON(), { etag })), cache)
   }
+
+  return { index, data }
 }
 
-function isEmpty(obj) {
-  if (!obj) return true
+loadIndex().catch(console.error)
 
-  for (const prop in obj) {
-    if (Object.prototype.hasOwnProperty.call(obj, prop)) { return false }
+onmessage = function(e) {
+  if (search) {
+    postMessage({ e: 'query-ready', d: search(e.data.q) })
   }
-
-  return true
 }
diff --git a/templates/package-lock.json b/templates/package-lock.json
diff --git a/templates/package.json b/templates/package.json
@@ -30,6 +30,7 @@
     "bootstrap": "^5.3.2",
     "bootstrap-icons": "^1.11.1",
     "highlight.js": "^11.8.0",
+    "idb-keyval": "^6.2.1",
     "jquery": "3.7.0",
     "lit-html": "^2.8.0",
     "lunr": "2.3.9",
@@ -38,6 +39,7 @@
   },
   "devDependencies": {
     "@types/jest": "^29.5.5",
+    "@types/lunr": "^2.3.5",
     "@typescript-eslint/eslint-plugin": "^6.7.4",
     "@typescript-eslint/parser": "^6.7.4",
     "browser-sync": "^2.29.3",

diff --git a/test/docfx.Snapshot.Tests/SamplesTest.Extensions/search-stopwords.verified.json b/test/docfx.Snapshot.Tests/SamplesTest.Extensions/search-stopwords.verified.json