Optimize serialization

CHANGELOG.md

@@ -8,6 +8,7 @@
 
 *2018-11-29*
 
+  * Optimize serialization and properly handle unicode in filters [#61](https://github.com/cliqz-oss/adblocker/pull/61)
   * Fix fuzzy matching by allowing tokens of any size [#61](https://github.com/cliqz-oss/adblocker/pull/62)
   * Add support for CSP (Content Security Policy) filters [#60](https://github.com/cliqz-oss/adblocker/pull/60)
   * Add hard-coded circumvention logic (+ IL defuser) [#59](https://github.com/cliqz-oss/adblocker/pull/59)

bench/micro.js

@@ -1,4 +1,4 @@
-const adblocker = require('../dist/adblocker.umd.min.js');
+const adblocker = require('../dist/adblocker.cjs.js');
 const { createEngine } = require('./utils');
 
 

bench/utils.js

@@ -1,5 +1,5 @@
 const fs = require('fs');
-const adblocker = require('../dist/adblocker.umd.min.js');
+const adblocker = require('../dist/adblocker.cjs.js');
 
 function createEngine(lists, resources, options = {}, serialize = false) {
   const engine = new adblocker.FiltersEngine({

example/background.ts

@@ -83,39 +83,56 @@ chrome.tabs.onActivated.addListener(({ tabId }) => {
   updateBadgeCount(tabId);
 });
 
-loadAdblocker().then((engine) => {
-  function listener({ tabId, type, url }) {
-    let source;
-    if (tabs.has(tabId)) {
-      source = tabs.get(tabId).source;
-    }
-    const result = engine.match({
-      sourceUrl: source,
-      type,
-      url,
-    });
-
-    if (result.redirect) {
-      incrementBlockedCounter(tabId);
-      return { redirectUrl: result.redirect };
-    } else if (result.match) {
-      incrementBlockedCounter(tabId);
-      return { cancel: true };
-    }
-
-    return {};
+function requestFromDetails({ tabId, type, url }) {
+  let source;
+  if (tabs.has(tabId)) {
+    source = tabs.get(tabId).source;
   }
+  return {
+    sourceUrl: source,
+    type,
+    url,
+  };
+}
 
+loadAdblocker().then((engine) => {
   // Start listening to requests, and allow 'blocking' so that we can cancel
   // some of them (or redirect).
   chrome.webRequest.onBeforeRequest.addListener(
-    listener,
+    (details) => {
+      const result = engine.match(requestFromDetails(details));
+
+      if (result.redirect) {
+        incrementBlockedCounter(details.tabId);
+        return { redirectUrl: result.redirect };
+      } else if (result.match) {
+        incrementBlockedCounter(details.tabId);
+        return { cancel: true };
+      }
+
+      return {};
+    },
     {
-      urls: ['*://*/*'],
+      urls: ['<all_urls>'],
     },
     ['blocking'],
   );
 
+  chrome.webRequest.onHeadersReceived.addListener(
+    (details) => {
+      if (details.type !== 'main_frame') {
+        return {};
+      }
+
+      return adblocker.updateResponseHeadersWithCSP(
+        details,
+        engine.getCSPDirectives(requestFromDetails(details)),
+      );
+    },
+    { urls: ['<all_urls>'] },
+    ['blocking', 'responseHeaders'],
+  );
+
   // Start listening to messages coming from the content-script
   chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
     // Extract hostname from sender's URL

example/rollup.config.js

@@ -3,7 +3,9 @@ import commonjs from 'rollup-plugin-commonjs';
 
 
 const plugins = [
-  resolve(),
+  resolve({
+    preferBuiltins: false,
+  }),
   commonjs(),
 ];
 

index.ts

@@ -17,4 +17,4 @@ export { f, parseList } from './src/parsing/list';
 export { compactTokens, hasEmptyIntersection, mergeCompactSets } from './src/compact-set';
 
 export { fetchLists, fetchResources } from './src/fetch';
-export { tokenize, fastHash } from './src/utils';
+export { tokenize, fastHash, updateResponseHeadersWithCSP } from './src/utils';

package.json

@@ -56,6 +56,7 @@
     "typescript": "^3.1.6"
   },
   "dependencies": {
+    "punycode": "^2.1.1",
     "tldts": "^3.0.0",
     "tslib": "^1.9.3"
   }

rollup.config.js

@@ -3,7 +3,9 @@ import commonjs from 'rollup-plugin-commonjs';
 import pkg from './package.json';
 
 const plugins = [
-  resolve(),
+  resolve({
+    preferBuiltins: false,
+  }),
   commonjs(),
 ];
 

src/data-view.ts

@@ -0,0 +1,169 @@
+import * as punycode from 'punycode';
+import { hasUnicode } from './utils';
+
+/**
+ * @class StaticDataView
+ *
+ * This abstraction allows to serialize efficiently low-level values of types:
+ * String, uint8, uint16, uint32 while hiding the complexity of managing the
+ * current offset and growing. It should always be instantiated with a
+ * big-enough length because this will not allow for resizing.
+ *
+ * This class is also more efficient than the built-in `DataView`.
+ *
+ * The way this is used in practice is that you write pairs of function to
+ * serialize (respectively) deserialize a given structure/class (with code being
+ * pretty symetrical). In the serializer you `pushX` values, and in the
+ * deserializer you use `getX` functions to get back the values.
+ */
+export default class StaticDataView {
+  protected buffer: Uint8Array;
+  protected pos: number;
+
+  constructor(length: number, buffer?: Uint8Array) {
+    this.buffer = buffer !== undefined ? buffer : new Uint8Array(length);
+    this.pos = 0;
+  }
+
+  public seekZero(): void {
+    this.pos = 0;
+  }
+
+  public crop(): Uint8Array {
+    if (this.pos >= this.buffer.byteLength) {
+      throw new Error(
+        `StaticDataView too small: ${this.buffer.byteLength}, but required ${this.pos - 1} bytes`,
+      );
+    }
+    return this.buffer.subarray(0, this.pos);
+  }
+
+  public set(buffer: Uint8Array): void {
+    this.buffer = new Uint8Array(buffer);
+    this.seekZero();
+  }
+
+  public pushByte(octet: number): void {
+    this.pushUint8(octet);
+  }
+
+  public getByte(): number {
+    return this.getUint8();
+  }
+
+  public pushUint8(uint8: number): void {
+    this.buffer[this.pos] = uint8;
+    this.pos += 1;
+  }
+
+  public getUint8(): number {
+    const uint8 = this.buffer[this.pos];
+    this.pos += 1;
+    return uint8;
+  }
+
+  public pushUint16(uint16: number): void {
+    this.buffer[this.pos] = uint16 >>> 8;
+    this.buffer[this.pos + 1] = uint16;
+    this.pos += 2;
+  }
+
+  public getUint16(): number {
+    const uint16 = ((this.buffer[this.pos] << 8) | this.buffer[this.pos + 1]) >>> 0;
+    this.pos += 2;
+    return uint16;
+  }
+
+  public pushUint32(uint32: number): void {
+    this.buffer[this.pos] = uint32 >>> 24;
+    this.buffer[this.pos + 1] = uint32 >>> 16;
+    this.buffer[this.pos + 2] = uint32 >>> 8;
+    this.buffer[this.pos + 3] = uint32;
+    this.pos += 4;
+  }
+
+  public pushUint32Array(arr: Uint32Array | undefined): void {
+    if (arr === undefined) {
+      this.pushUint16(0);
+    } else {
+      this.pushUint16(arr.length);
+      for (let i = 0; i < arr.length; i += 1) {
+        this.pushUint32(arr[i]);
+      }
+    }
+  }
+
+  public getUint32Array(): Uint32Array | undefined {
+    const length = this.getUint16();
+    if (length === 0) {
+      return undefined;
+    }
+    const arr = new Uint32Array(length);
+    for (let i = 0; i < length; i += 1) {
+      arr[i] = this.getUint32();
+    }
+    return arr;
+  }
+
+  public getUint32(): number {
+    const uint32 =
+      (((this.buffer[this.pos] << 24) >>> 0) +
+        ((this.buffer[this.pos + 1] << 16) |
+          (this.buffer[this.pos + 2] << 8) |
+          this.buffer[this.pos + 3])) >>>
+      0;
+    this.pos += 4;
+    return uint32;
+  }
+
+  public pushUTF8(str: string | undefined): void {
+    if (str === undefined) {
+      this.pushUint16(0);
+    } else {
+      this.pushUint16(str.length);
+      if (hasUnicode(str)) {
+        this.pushASCII(punycode.encode(str));
+      } else {
+        this.pushASCII(str);
+      }
+    }
+  }
+
+  public getUTF8(): string | undefined {
+    const length = this.getUint16();
+    if (length === 0) {
+      return undefined;
+    }
+
+    const str = this.getASCII();
+    if (str === undefined || str.length === length) {
+      return str;
+    }
+    return punycode.decode(str);
+  }
+
+  public pushASCII(str: string | undefined): void {
+    if (str === undefined) {
+      this.pushUint16(0);
+    } else {
+      this.pushUint16(str.length);
+      const len = str.length;
+      const offset = this.pos;
+      for (let i = 0; i < len; i += 1) {
+        this.buffer[offset + i] = str.charCodeAt(i);
+      }
+      this.pos += len;
+    }
+  }
+
+  public getASCII(): string | undefined {
+    const byteLength = this.getUint16();
+
+    if (byteLength === 0) {
+      return undefined;
+    }
+
+    this.pos += byteLength;
+    return String.fromCharCode.apply(null, this.buffer.subarray(this.pos - byteLength, this.pos));
+  }
+}