From 02fbfbb673f94b1051d9d46d9327b7264a3cebe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= Date: Wed, 7 Aug 2019 22:14:28 +0200 Subject: [PATCH] electron: fix bundles (#249) --- packages/adblocker-electron-example/index.ts | 74 ++++++++++++------- .../adblocker-electron-example/package.json | 6 +- packages/adblocker-electron/adblocker.ts | 2 +- .../{content.ts => preload.ts} | 4 +- packages/adblocker-electron/rollup.config.ts | 36 +++++---- packages/adblocker-electron/tsconfig.json | 2 +- packages/adblocker/src/engine/engine.ts | 2 +- yarn.lock | 9 ++- 8 files changed, 88 insertions(+), 47 deletions(-) rename packages/adblocker-electron/{content.ts => preload.ts} (98%) diff --git a/packages/adblocker-electron-example/index.ts b/packages/adblocker-electron-example/index.ts index 1ba32a6713..e91d3b67ef 100644 --- a/packages/adblocker-electron-example/index.ts +++ b/packages/adblocker-electron-example/index.ts @@ -1,33 +1,55 @@ -import axios from 'axios'; import { app, BrowserWindow, session } from 'electron'; +import fetch from 'node-fetch'; -import { ElectronBlocker, ENGINE_VERSION } from '@cliqz/adblocker-electron'; +import { ElectronBlocker, fetchLists, fetchResources } from '@cliqz/adblocker-electron'; +// Polyfill fetch API for Node.js environment +// @ts-ignore +global.fetch = fetch; + +/** + * Initialize the adblocker using lists of filters and resources. It returns a + * Promise resolving on the `Engine` that we will use to decide what requests + * should be blocked or altered. + */ async function loadAdblocker(): Promise { - // Fetch `allowed-lists.json` from CDN. It contains information about where - // to find pre-built engines as well as lists of filters (e.g.: Easylist, - // etc.). - console.time('fetch allowed lists'); - const { engines } = (await axios.get( - 'https://cdn.cliqz.com/adblocker/configs/desktop-ads-trackers/allowed-lists.json', - )).data; - console.timeEnd('fetch allowed lists'); - - // Once we have the config, we can get the URL of the pre-built engine - // corresponding to our installed @cliqz/adblocker version (i.e.: - // ENGINE_VERSION). This guarantees that we can download a compabitle one. - console.time('fetch serialized engine'); - const serialized = (await axios.get(engines[ENGINE_VERSION].url, { - responseType: 'arraybuffer', - })).data; - console.timeEnd('fetch serialized engine'); - - // Deserialize the FiltersEngine instance from binary form. - console.time('deserialize engine'); - const engine = ElectronBlocker.deserialize(new Uint8Array(serialized)) as ElectronBlocker; - console.timeEnd('deserialize engine'); - - return engine; + console.log('Fetching resources...'); + return Promise.all([fetchLists(), fetchResources()]).then(([responses, resources]) => { + console.log('Initialize adblocker...'); + const deduplicatedLines = new Set(); + for (let i = 0; i < responses.length; i += 1) { + const lines = responses[i].split(/\n/g); + for (let j = 0; j < lines.length; j += 1) { + deduplicatedLines.add(lines[j]); + } + } + const deduplicatedFilters = Array.from(deduplicatedLines).join('\n'); + + let t0 = Date.now(); + const engine = ElectronBlocker.parse(deduplicatedFilters, { + enableCompression: true, + }); + let total = Date.now() - t0; + console.log('parsing filters', total); + + t0 = Date.now(); + engine.updateResources(resources, '' + resources.length); + total = Date.now() - t0; + console.log('parsing resources', total); + + t0 = Date.now(); + const serialized = engine.serialize(); + total = Date.now() - t0; + console.log('serialization', total); + console.log('size', serialized.byteLength); + + t0 = Date.now(); + const deserialized = ElectronBlocker.deserialize(serialized); + total = Date.now() - t0; + console.log('deserialization', total); + + return deserialized as ElectronBlocker; + }); } let mainWindow: BrowserWindow | null = null; diff --git a/packages/adblocker-electron-example/package.json b/packages/adblocker-electron-example/package.json index 4a02f77db2..a2179027d7 100644 --- a/packages/adblocker-electron-example/package.json +++ b/packages/adblocker-electron-example/package.json @@ -8,7 +8,8 @@ "license": "MPL-2.0", "files": [ "LICENSE", - "index.ts" + "index.ts", + "index.js" ], "repository": { "type": "git", @@ -23,8 +24,9 @@ }, "dependencies": { "@cliqz/adblocker-electron": "^0.12.0", - "axios": "^0.19.0", + "@types/node-fetch": "^2.5.0", "electron": "^5.0.7", + "node-fetch": "^2.6.0", "ts-node": "^8.3.0", "typescript": "^3.5.3" } diff --git a/packages/adblocker-electron/adblocker.ts b/packages/adblocker-electron/adblocker.ts index 69fa5d7b9f..3b82855f88 100644 --- a/packages/adblocker-electron/adblocker.ts +++ b/packages/adblocker-electron/adblocker.ts @@ -48,7 +48,7 @@ export class ElectronBlocker extends FiltersEngine { ses.webRequest.onBeforeRequest({ urls: [''] }, this.onBeforeRequest); ipcMain.on('get-cosmetic-filters', this.onGetCosmeticFilters); - ses.setPreloads([join(__dirname, './content.js')]); + ses.setPreloads([join(__dirname, './preload.js')]); ipcMain.on('is-mutation-observer-enabled', (event: Electron.IpcMessageEvent) => { event.returnValue = this.config.enableMutationObserver; diff --git a/packages/adblocker-electron/content.ts b/packages/adblocker-electron/preload.ts similarity index 98% rename from packages/adblocker-electron/content.ts rename to packages/adblocker-electron/preload.ts index f8fa1fdc52..51d83f40e0 100644 --- a/packages/adblocker-electron/content.ts +++ b/packages/adblocker-electron/preload.ts @@ -41,8 +41,8 @@ function handleResponseFromBackground({ active, scripts }: IMessageFromBackgroun ACTIVE = true; } - for (const script of scripts) { - setTimeout(() => webFrame.executeJavaScript(script), 1); + for (let i = 0; i < scripts.length; i += 1) { + setTimeout(() => webFrame.executeJavaScript(scripts[i]), 1); } } diff --git a/packages/adblocker-electron/rollup.config.ts b/packages/adblocker-electron/rollup.config.ts index 0e7baa8804..3c89cd7240 100644 --- a/packages/adblocker-electron/rollup.config.ts +++ b/packages/adblocker-electron/rollup.config.ts @@ -9,17 +9,27 @@ import resolve from 'rollup-plugin-node-resolve'; import sourcemaps from 'rollup-plugin-sourcemaps'; -export default { - // CommonJs bundle for preload script - external: ['electron'], - input: './dist/es6/content.js', - output: { - file: './dist/content.js', - format: 'cjs', - sourcemap: true, +export default [ + { + // CommonJs bundle for preload script + external: ['electron', 'path'], + input: './dist/es6/adblocker.js', + output: { + file: './dist/cjs/adblocker.js', + format: 'cjs', + sourcemap: true, + }, + plugins: [resolve({ preferBuiltins: true }), sourcemaps()], }, - plugins: [ - resolve(), - sourcemaps(), - ], -}; + { + // CommonJs bundle for preload script + external: ['electron'], + input: './dist/es6/preload.js', + output: { + file: './dist/cjs/preload.js', + format: 'cjs', + sourcemap: true, + }, + plugins: [resolve({ preferBuiltins: true }), sourcemaps()], + }, +]; diff --git a/packages/adblocker-electron/tsconfig.json b/packages/adblocker-electron/tsconfig.json index 7be3ea5c9f..a332e4984b 100644 --- a/packages/adblocker-electron/tsconfig.json +++ b/packages/adblocker-electron/tsconfig.json @@ -11,6 +11,6 @@ ], "files": [ "adblocker.ts", - "content.ts" + "preload.ts" ] } diff --git a/packages/adblocker/src/engine/engine.ts b/packages/adblocker/src/engine/engine.ts index 28d38f455b..a6e823df6b 100644 --- a/packages/adblocker/src/engine/engine.ts +++ b/packages/adblocker/src/engine/engine.ts @@ -16,7 +16,7 @@ import Resources from '../resources'; import CosmeticFilterBucket from './bucket/cosmetic'; import NetworkFilterBucket from './bucket/network'; -export const ENGINE_VERSION = 30; +export const ENGINE_VERSION = 31; // Polyfill for `btoa` function btoaPolyfill(buffer: string): string { diff --git a/yarn.lock b/yarn.lock index 0427b9dc2f..4aa894f16b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1378,6 +1378,13 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== +"@types/node-fetch@^2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.0.tgz#1c55616a4591bdd15a389fbd0da4a55b9502add5" + integrity sha512-TLFRywthBgL68auWj+ziWu+vnmmcHCDFC/sqCOQf1xTz4hRq8cu79z8CtHU9lncExGBsB8fXA4TiLDLt6xvMzw== + dependencies: + "@types/node" "*" + "@types/node@*", "@types/node@^12.0.2", "@types/node@^12.0.3", "@types/node@^12.6.9": version "12.7.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.0.tgz#545dde2a1a5c27d281cfb8308d6736e0708f5d6c" @@ -5305,7 +5312,7 @@ node-fetch-npm@^2.0.2: json-parse-better-errors "^1.0.0" safe-buffer "^5.1.1" -node-fetch@^2.3.0, node-fetch@^2.5.0: +node-fetch@^2.3.0, node-fetch@^2.5.0, node-fetch@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==