electron: fix bundles (#249)

packages/adblocker-electron-example/index.ts

@@ -1,33 +1,55 @@
-import axios from 'axios';
 import { app, BrowserWindow, session } from 'electron';
+import fetch from 'node-fetch';
 
-import { ElectronBlocker, ENGINE_VERSION } from '@cliqz/adblocker-electron';
+import { ElectronBlocker, fetchLists, fetchResources } from '@cliqz/adblocker-electron';
 
+// Polyfill fetch API for Node.js environment
+// @ts-ignore
+global.fetch = fetch;
+
+/**
+ * Initialize the adblocker using lists of filters and resources. It returns a
+ * Promise resolving on the `Engine` that we will use to decide what requests
+ * should be blocked or altered.
+ */
 async function loadAdblocker(): Promise<ElectronBlocker> {
-  // Fetch `allowed-lists.json` from CDN. It contains information about where
-  // to find pre-built engines as well as lists of filters (e.g.: Easylist,
-  // etc.).
-  console.time('fetch allowed lists');
-  const { engines } = (await axios.get(
-    'https://cdn.cliqz.com/adblocker/configs/desktop-ads-trackers/allowed-lists.json',
-  )).data;
-  console.timeEnd('fetch allowed lists');
-
-  // Once we have the config, we can get the URL of the pre-built engine
-  // corresponding to our installed @cliqz/adblocker version (i.e.:
-  // ENGINE_VERSION). This guarantees that we can download a compabitle one.
-  console.time('fetch serialized engine');
-  const serialized = (await axios.get(engines[ENGINE_VERSION].url, {
-    responseType: 'arraybuffer',
-  })).data;
-  console.timeEnd('fetch serialized engine');
-
-  // Deserialize the FiltersEngine instance from binary form.
-  console.time('deserialize engine');
-  const engine = ElectronBlocker.deserialize(new Uint8Array(serialized)) as ElectronBlocker;
-  console.timeEnd('deserialize engine');
-
-  return engine;
+  console.log('Fetching resources...');
+  return Promise.all([fetchLists(), fetchResources()]).then(([responses, resources]) => {
+    console.log('Initialize adblocker...');
+    const deduplicatedLines = new Set();
+    for (let i = 0; i < responses.length; i += 1) {
+      const lines = responses[i].split(/\n/g);
+      for (let j = 0; j < lines.length; j += 1) {
+        deduplicatedLines.add(lines[j]);
+      }
+    }
+    const deduplicatedFilters = Array.from(deduplicatedLines).join('\n');
+
+    let t0 = Date.now();
+    const engine = ElectronBlocker.parse(deduplicatedFilters, {
+      enableCompression: true,
+    });
+    let total = Date.now() - t0;
+    console.log('parsing filters', total);
+
+    t0 = Date.now();
+    engine.updateResources(resources, '' + resources.length);
+    total = Date.now() - t0;
+    console.log('parsing resources', total);
+
+    t0 = Date.now();
+    const serialized = engine.serialize();
+    total = Date.now() - t0;
+    console.log('serialization', total);
+    console.log('size', serialized.byteLength);
+
+    t0 = Date.now();
+    const deserialized = ElectronBlocker.deserialize(serialized);
+    total = Date.now() - t0;
+    console.log('deserialization', total);
+
+    return deserialized as ElectronBlocker;
+  });
 }
 
 let mainWindow: BrowserWindow | null = null;

packages/adblocker-electron-example/package.json

@@ -8,7 +8,8 @@
   "license": "MPL-2.0",
   "files": [
     "LICENSE",
-    "index.ts"
+    "index.ts",
+    "index.js"
   ],
   "repository": {
     "type": "git",
@@ -23,8 +24,9 @@
   },
   "dependencies": {
     "@cliqz/adblocker-electron": "^0.12.0",
-    "axios": "^0.19.0",
+    "@types/node-fetch": "^2.5.0",
     "electron": "^5.0.7",
+    "node-fetch": "^2.6.0",
     "ts-node": "^8.3.0",
     "typescript": "^3.5.3"
   }

packages/adblocker-electron/adblocker.ts

@@ -48,7 +48,7 @@ export class ElectronBlocker extends FiltersEngine {
     ses.webRequest.onBeforeRequest({ urls: ['<all_urls>'] }, this.onBeforeRequest);
 
     ipcMain.on('get-cosmetic-filters', this.onGetCosmeticFilters);
-    ses.setPreloads([join(__dirname, './content.js')]);
+    ses.setPreloads([join(__dirname, './preload.js')]);
 
     ipcMain.on('is-mutation-observer-enabled', (event: Electron.IpcMessageEvent) => {
       event.returnValue = this.config.enableMutationObserver;

packages/adblocker-electron/content.ts → packages/adblocker-electron/preload.ts

@@ -41,8 +41,8 @@ function handleResponseFromBackground({ active, scripts }: IMessageFromBackgroun
     ACTIVE = true;
   }
 
-  for (const script of scripts) {
-    setTimeout(() => webFrame.executeJavaScript(script), 1);
+  for (let i = 0; i < scripts.length; i += 1) {
+    setTimeout(() => webFrame.executeJavaScript(scripts[i]), 1);
   }
 }
 

packages/adblocker-electron/rollup.config.ts

@@ -9,17 +9,27 @@
 import resolve from 'rollup-plugin-node-resolve';
 import sourcemaps from 'rollup-plugin-sourcemaps';
 
-export default {
-  // CommonJs bundle for preload script
-  external: ['electron'],
-  input: './dist/es6/content.js',
-  output: {
-    file: './dist/content.js',
-    format: 'cjs',
-    sourcemap: true,
+export default [
+  {
+    // CommonJs bundle for preload script
+    external: ['electron', 'path'],
+    input: './dist/es6/adblocker.js',
+    output: {
+      file: './dist/cjs/adblocker.js',
+      format: 'cjs',
+      sourcemap: true,
+    },
+    plugins: [resolve({ preferBuiltins: true }), sourcemaps()],
   },
-  plugins: [
-    resolve(),
-    sourcemaps(),
-  ],
-};
+  {
+    // CommonJs bundle for preload script
+    external: ['electron'],
+    input: './dist/es6/preload.js',
+    output: {
+      file: './dist/cjs/preload.js',
+      format: 'cjs',
+      sourcemap: true,
+    },
+    plugins: [resolve({ preferBuiltins: true }), sourcemaps()],
+  },
+];

packages/adblocker-electron/tsconfig.json

@@ -11,6 +11,6 @@
   ],
   "files": [
     "adblocker.ts",
-    "content.ts"
+    "preload.ts"
   ]
 }

packages/adblocker/src/engine/engine.ts

@@ -16,7 +16,7 @@ import Resources from '../resources';
 import CosmeticFilterBucket from './bucket/cosmetic';
 import NetworkFilterBucket from './bucket/network';
 
-export const ENGINE_VERSION = 30;
+export const ENGINE_VERSION = 31;
 
 // Polyfill for `btoa`
 function btoaPolyfill(buffer: string): string {

yarn.lock

@@ -1378,6 +1378,13 @@
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
   integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==
 
+"@types/node-fetch@^2.5.0":
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.0.tgz#1c55616a4591bdd15a389fbd0da4a55b9502add5"
+  integrity sha512-TLFRywthBgL68auWj+ziWu+vnmmcHCDFC/sqCOQf1xTz4hRq8cu79z8CtHU9lncExGBsB8fXA4TiLDLt6xvMzw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/node@*", "@types/node@^12.0.2", "@types/node@^12.0.3", "@types/node@^12.6.9":
   version "12.7.0"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.0.tgz#545dde2a1a5c27d281cfb8308d6736e0708f5d6c"
@@ -5305,7 +5312,7 @@ node-fetch-npm@^2.0.2:
     json-parse-better-errors "^1.0.0"
     safe-buffer "^5.1.1"
 
-node-fetch@^2.3.0, node-fetch@^2.5.0:
+node-fetch@^2.3.0, node-fetch@^2.5.0, node-fetch@^2.6.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
   integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==