feat: hmr example

package.json

@@ -115,6 +115,8 @@
     "@types/enzyme-adapter-react-16": "^1.0.6",
     "@types/expect-puppeteer": "^4.4.3",
     "@types/gulp": "^4.0.6",
+    "@types/gulp-changed": "^0.0.33",
+    "@types/gulp-rename": "^0.0.33",
     "@types/gulp-zip": "^4.0.1",
     "@types/gun": "^0.9.2",
     "@types/jest": "^26.0.3",
@@ -130,10 +132,12 @@
     "@types/react-router-dom": "^5.1.5",
     "@types/react-virtualized-auto-sizer": "^1.0.0",
     "@types/react-window": "^1.8.2",
+    "@types/rimraf": "^3.0.0",
     "@types/tiny-secp256k1": "^1.0.0",
     "@types/use-subscription": "^1.0.0",
     "@types/uuid": "^8.0.0",
     "@types/webpack": "^4.41.18",
+    "@types/ws": "^7.2.6",
     "@typescript-eslint/eslint-plugin": "^3.4.0",
     "@typescript-eslint/parser": "^3.5.0",
     "babel-loader": "^8.1.0",
@@ -150,6 +154,7 @@
     "expect-puppeteer": "^4.4.0",
     "fake-indexeddb": "^3.0.2",
     "gulp": "^4.0.2",
+    "gulp-changed": "^4.0.2",
     "gulp-multi-process": "^1.4.0",
     "gulp-rename": "^2.0.0",
     "gulp-zip": "^5.0.2",
@@ -179,7 +184,8 @@
     "typescript": "^3.9.7",
     "web-ext": "^4.3.0",
     "web-ext-types": "^3.1.0",
-    "webpack": "^4.43.0"
+    "webpack": "^4.43.0",
+    "ws": "^7.3.1"
   },
   "optionalDependencies": {
     "expect-puppeteer": "^4.4.0",

scripts/gulp/assets.ts

@@ -1,63 +1,64 @@
-import { src, dest, watch, lastRun } from 'gulp'
+import { src, dest, watch } from 'gulp'
 import * as modifier from './manifest.overrides'
-import { createTask, modifyFile, named } from './helper'
+import { copyOnChange, createTask, modifyFile, named } from './helper'
 import { assetsPath, output, manifestPath, srcPath } from './paths'
 import { buildTarget, getEnvironment } from './env'
-// @ts-ignore
 import rename from 'gulp-rename'
+import changed from 'gulp-changed'
 
-const sourceAssetsDesc =
-    'Copy all assets allocated in src/* to extension/esm/* to enable them to be used by import.meta'
-export function srcAssets() {
-    return src(
+export const [srcAssets, watchSrcAssets] = copyOnChange({
+    name: 'src-assets',
+    desc: 'Copy all assets to the extension folder',
+    from: [
         [
             srcPath.files,
             `!${srcPath.relative('./**/*.ts')}`,
             `!${srcPath.relative('./**/*.tsx')}`,
             `!${srcPath.relative('./**/*.js')}`,
         ],
-        {
-            since: lastRun(srcAssets),
-        },
-    ).pipe(dest(output.esmBuildClone.folder))
-}
-named('src-assets', sourceAssetsDesc + ' (build)', srcAssets)
-export const watchSrcAssets = named('watch-src-assets', sourceAssetsDesc + ' (watch)', () =>
-    watch(srcPath.folder, { ignoreInitial: true, ignored: ['*.ts', '*.tsx'] }, srcAssets),
-)
+    ],
+    to: output.esmBuildClone.folder,
+    watch: [srcPath.folder, { ignored: ['*.ts', '*.tsx'] }],
+})
 
-export function assets() {
-    return src(assetsPath.files, { since: lastRun(assets) }).pipe(dest(output.extension.folder))
-}
-named(assets.name, 'Copy all assets to the extension folder (build)', assets)
-export const watchAssets = named('watch-assets', 'Copy all assets to the extension folder (watch)', () =>
-    watch(assetsPath.folder, { ignoreInitial: false }, assets),
-)
+export const [assets, watchAssets] = copyOnChange({
+    name: 'assets',
+    desc: 'Copy all assets to the extension folder',
+    from: [assetsPath.files],
+    to: output.extension.folder,
+    watch: [assetsPath.folder],
+})
 
 const modify = (watch: boolean) => (x: string): string => {
     const obj = JSON.parse(x)
     if (watch) modifier.development(obj)
+    else modifier.production(obj)
     modifier[buildTarget](obj)
     return JSON.stringify(obj, void 0, 4)
 }
 export function manifest() {
     return src(manifestPath.file)
         .pipe(modifyFile(modify(false)))
+        .pipe(changed(output.extension.folder))
         .pipe(dest(output.extension.folder))
 }
 named(manifest.name, 'Generate the extension manifest based on the build target (build)', manifest)
 export const watchManifest = named(
     'watch-manifest',
     'Generate the extension manifest based on the build target (watch)',
     () =>
-        watch(manifestPath.file, { ignoreInitial: false }, function watch_manifest_inner() {
-            return src(manifestPath.file)
-                .pipe(modifyFile(modify(false)))
-                .pipe(dest(output.extension.folder))
+        watch(manifestPath.file, { ignoreInitial: false }, function manifest() {
+            return (
+                src(manifestPath.file)
+                    // Notify the true. this is different than the fn manifest above.
+                    .pipe(modifyFile(modify(true)))
+                    .pipe(changed(output.extension.folder))
+                    .pipe(dest(output.extension.folder))
+            )
         }),
 )
 
-export const { build: environmentFile, watch: watchEnvironmentFile } = createTask(
+export const [environmentFile, watchEnvironmentFile] = createTask(
     'environment-file',
     'Create a env.js in the output folder for environment variables',
     (mode) => () =>
@@ -69,5 +70,6 @@ globalThis.process.env = ${JSON.stringify(getEnvironment(mode))};`,
                 ),
             )
             .pipe(rename('env.js'))
+            .pipe(changed(output.extension.folder))
             .pipe(dest(output.extension.folder)),
 )

scripts/gulp/build-iOS.ts

@@ -3,7 +3,6 @@ import { buildArchitecture, buildTarget } from './env'
 import { output } from './paths'
 import { gulpPrebuilt } from '@dimensiondev/webextension-shim/dist/bin/prebuilt-lib'
 import { named } from './helper'
-// @ts-ignore
 import rimraf from 'rimraf'
 import { promisify } from 'util'
 export function prebuilt_iOS(done: any) {

scripts/gulp/build-isolated.ts

@@ -1,6 +1,6 @@
 import { buildWebpackTask, getWebpackConfig } from './helper'
 import { output, IsolatedEntries } from './paths'
-export const { build: isolatedBuild, watch: isolatedWatch } = buildWebpackTask(
+export const [isolatedBuild, isolatedWatch] = buildWebpackTask(
     'isolated',
     'Build isolated entries (injected scripts) by Webpack',
     (mode) => {

scripts/gulp/build-worker.ts

@@ -7,12 +7,8 @@
  */
 import { buildWebpackTask, getWebpackConfig } from './helper'
 import { output, WorkerEntries } from './paths'
-export const { build: workerBuild, watch: workerWatch } = buildWebpackTask(
-    'workers',
-    'Build Web Workers by Webpack',
-    (mode) => {
-        const conf = getWebpackConfig(mode, WorkerEntries, output.workers.folder)
-        conf.target = 'webworker'
-        return conf
-    },
-)
+export const [workerBuild, workerWatch] = buildWebpackTask('workers', 'Build Web Workers by Webpack', (mode) => {
+    const conf = getWebpackConfig(mode, WorkerEntries, output.workers.folder)
+    conf.target = 'webworker'
+    return conf
+})

scripts/gulp/dependencies.ts

@@ -1,14 +1,37 @@
 import { readFileSync } from 'fs'
 import HtmlWebpackPlugin from 'html-webpack-plugin'
 import type { Configuration } from 'webpack'
-import { buildWebpackTask, getWebpackConfig } from './helper'
+import { buildWebpackTask, copyOnChange, getWebpackConfig } from './helper'
 import { assetsPath, entries, output } from './paths'
 
-export const { watch: dependenciesWatch, build: dependenciesBuild } = buildWebpackTask(
+const runtimeOut = output.libraries.relativeFolder('./bundle/')
+const [, copyWebpackOut1] = copyOnChange({
+    name: 'copy-webpack-output',
+    desc: 'Copy webpack output',
+    from: [output.webpackDependenciesJS.files],
+    to: output.librariesBundle.folder,
+    watch: [output.webpackDependenciesJS.folder],
+})
+const [, copyWebpackOut2] = copyOnChange({
+    name: 'copy-webpack-output',
+    desc: 'Copy webpack output',
+    from: [output.webpackDependenciesHTML.relative('./*.html')],
+    to: output.extension.folder,
+    watch: [output.webpackDependenciesHTML.folder],
+})
+export const [dependenciesBuild, dependenciesWatch] = buildWebpackTask(
     'dependencies',
     'Build all node style dependencies by Webpack',
     (mode) => {
-        const obj = getWebpackConfig(mode, entries, output.libraries.relative('./bundle/'))
+        const obj = getWebpackConfig(
+            mode,
+            entries,
+            mode === 'development' ? output.webpackDependenciesJS.folder : runtimeOut.folder,
+        )
+        if (mode === 'development') {
+            copyWebpackOut1(() => {})
+            copyWebpackOut2(() => {})
+        }
         obj.output!.publicPath = output.libraries.relativeFromRuntimeExtensionRoot('./bundle/')
         // replace ts-loader
         obj.module!.rules[2] = {

scripts/gulp/helper.ts

@@ -1,10 +1,11 @@
 import { promisify } from 'util'
 import type { Configuration } from 'webpack'
 import webpack from 'webpack'
-import type { TaskFunction } from 'gulp'
+import { dest, lastRun, src, TaskFunction, watch } from 'gulp'
 import ts from 'typescript'
 import { getEnvironment } from './env'
 import { Readable, Transform } from 'stream'
+import changed from 'gulp-changed'
 
 export function modifyFile(fn: (x: string) => string) {
     const stream = new Transform({
@@ -90,13 +91,13 @@ export function buildWebpackTask(name: string, desc: string, config: (mode: Conf
     }
     watch.displayName = `watch-${name}`
     watch.description = `${desc} (watch)`
-    return { watch, build }
+    return [build, watch] as const
 }
 export function createTask(name: string, desc: string, f: (mode: Configuration['mode']) => TaskFunction) {
-    return {
-        watch: named(`watch-${name}`, `${desc} (watch)`, f('development')),
-        build: named(name, `${desc} (build)`, f('production')),
-    }
+    return [
+        named(name, `${desc} (build)`, f('production')),
+        named(`watch-${name}`, `${desc} (watch)`, f('development')),
+    ] as const
 }
 export function named(displayName: string, desc: string, f: TaskFunction) {
     f.displayName = displayName
@@ -111,3 +112,23 @@ export function toSystem(x: string) {
         },
     }).outputText
 }
+export function copyOnChange(opts: {
+    name: string
+    desc: string
+    from: Parameters<typeof src>
+    to: string
+    watch: Parameters<typeof watch>
+    transform?: (s: NodeJS.ReadWriteStream) => NodeJS.ReadWriteStream
+}) {
+    const { desc, name, from, to, watch: watchProps, transform } = opts
+    function copy() {
+        let s = src(from[0], { since: lastRun(copy), ...from[1] })
+        if (transform) s = transform(s)
+        return s.pipe(changed(to)).pipe(dest(to))
+    }
+    named(name, desc + ' (build)', copy)
+    const watchCopy = named('watch-' + name, desc + ' (watch)', () =>
+        watch(watchProps[0], { ignoreInitial: false, ...watchProps[1] }, copy),
+    )
+    return [copy, watchCopy] as const
+}

scripts/gulp/hmr.ts

@@ -0,0 +1,20 @@
+import WebSocket, { Server } from 'ws'
+import { watch } from 'gulp'
+import { output } from './paths'
+import { posix } from 'path'
+import { pathToFileURL } from 'url'
+export function hmrServer() {
+    const server = new Server({ port: 7687 })
+    const clients = new Set<WebSocket>()
+    server.addListener('connection', (client) => {
+        clients.add(client)
+        client.addListener('close', () => clients.delete(client))
+    })
+    const watcher = watch(output.extension.folder)
+    watcher.addListener('change', (fileName) => {
+        const url = pathToFileURL(fileName).href
+        const r = posix.relative(pathToFileURL(output.extension.folder).href, url)
+        console.log(r, 'changed')
+        for (const each of clients) each.send(r)
+    })
+}

scripts/gulp/index.ts

@@ -21,6 +21,8 @@ import { workerBuild, workerWatch } from './build-worker'
 import { isolatedBuild, isolatedWatch } from './build-isolated'
 import { prebuilt_iOS } from './build-iOS'
 import { buildTarget } from './env'
+import { hmrServer } from './hmr'
+import rimraf from 'rimraf'
 
 function parallelProcessWatch(done: any) {
     return gulpMultiProcess(
@@ -53,6 +55,7 @@ export const watch = named(
             isolatedWatch,
             watchCopyESMOut,
             parallelProcessWatch,
+            hmrServer,
         ),
     ),
 )
@@ -95,7 +98,7 @@ function parallelProcessBuild(done: any) {
     )
 }
 export function clean(cb: any) {
-    require('rimraf')(output.extension.folder, cb)
+    rimraf(output.extension.folder, cb)
 }
 named(clean.name!, 'Clean previous extension build', clean)
 
@@ -109,3 +112,4 @@ export * from './libraries'
 export * from './build-worker'
 export * from './build-iOS'
 export * from './build-ci'
+export * from './hmr'

scripts/gulp/paths.ts

@@ -31,9 +31,12 @@ export const output = {
     temp: folder('../../temp/'),
     extension: folder('../../temp/extension/'),
     esmBuildOriginal: folder('../../temp/esm/'),
+    webpackDependenciesHTML: folder('../../temp/webpack/'),
+    webpackDependenciesJS: folder('../../temp/webpack/libraries/bundle/'),
     esmBuildClone: folder('../../temp/extension/esm/'),
     systemBuild: folder('../../temp/extension/system/'),
     libraries: folder('../../temp/extension/libraries/'),
+    librariesBundle: folder('../../temp/extension/libraries/bundle/'),
     polyfills: folder('../../temp/extension/polyfills/'),
     loaders: folder('../../temp/extension/loaders/'),
     isolated: folder('../../temp/extension/isolated/'),

scripts/gulp/tree-shake-loader.ts

@@ -91,7 +91,7 @@ function createRegisterFromImportCall(node: CallExpression) {
     if (!isImportCall(node)) return createEmptyStatement()
     const arg0 = node.arguments[0]
     if (isStringLiteral(arg0) && arg0.text.startsWith('.')) return createImportDeclaration(void 0, void 0, void 0, arg0)
-    return createExpressionStatement(createRegisterCall(arg0, node))
+    return createEmptyStatement()
 }
 /**
  * export { x as y } from 'z'

scripts/gulp/tsc.ts

@@ -4,15 +4,16 @@ import { output, srcPath, tsconfigESMPath } from './paths'
 import { createTask, modifyFile, named, toSystem } from './helper'
 import { join } from 'path'
 import { buildTarget } from './env'
+import changed from 'gulp-changed'
 
 const typescriptCLI = join(require.resolve('ttypescript'), '../tsc.js')
-export const { build: tscESModuleBuild, watch: tscESModuleWatch } = createTask(
+export const [tscESModuleBuild, tscESModuleWatch] = createTask(
     'esm',
     'Build all TypeScript into browser ES Module',
     (mode) => () =>
         spawn(
             'node',
-            [typescriptCLI, '-b', '--preserveWatchOutput', tsconfigESMPath.file, mode === 'development' ? ' -w' : ''],
+            [typescriptCLI, '--preserveWatchOutput', '-p', tsconfigESMPath.file, mode === 'development' ? ' -w' : ''],
             {
                 stdio: 'inherit',
                 cwd: srcPath.relative('../'),
@@ -27,6 +28,7 @@ export const tscSystemBuild = named(
         if (buildTarget !== 'firefox') return done()
         return src(output.esmBuildOriginal.js, { since: lastRun(tscSystemBuild) })
             .pipe(modifyFile((x) => toSystem(x).replace('ttsclib.js', 'ttsclib-system.js')))
+            .pipe(changed(output.systemBuild.folder))
             .pipe(dest(output.systemBuild.folder))
     },
 )
@@ -35,9 +37,10 @@ export const tscSystemWatch = named(
     'Build all TypeScript into SystemJS format for Firefox (watch)',
     () => watch(output.esmBuildOriginal.folder, { ignoreInitial: false }, tscSystemBuild),
 )
-// We have to do the copy, cause https://bugzilla.mozilla.org/show_bug.cgi?id=1654463
 export function copyESMOut() {
-    return src(output.esmBuildOriginal.js, { since: lastRun(copyESMOut) }).pipe(dest(output.esmBuildClone.folder))
+    return src(output.esmBuildOriginal.js, { since: lastRun(copyESMOut) })
+        .pipe(changed(output.esmBuildClone.folder))
+        .pipe(dest(output.esmBuildClone.folder))
 }
 named('copy-esm-out', 'Copy files from tsc output (build)', copyESMOut)
 export const watchCopyESMOut = named('watch-copy-esm-out', 'Copy files from tsc output (watch)', () =>