From 8b16858cbd2817d4c3e3cc313010af1fe15a5413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi?= Date: Fri, 23 Nov 2018 16:18:01 +0100 Subject: [PATCH] Add hard-coded circumvention logic (+ IL defuser) (#59) * Simplify 'example' extension * Add circumvention module and entry-point in cosmetics injection * Clean-up cjs and esm bundles * Remove obsolete logic to override user-agent in content-script * Simplify travis config (using new pre* hooks) * Consolidate 'fetch' module (with metadata about lists) --- .travis.yml | 6 - CHANGELOG.md | 13 + example/Makefile | 21 +- example/README.md | 14 +- example/background.ts | 16 +- example/content-script.ts | 28 +- example/manifest.json | 4 +- example/rollup.background.js | 10 - example/rollup.config.js | 27 ++ example/rollup.content.js | 11 - example/tsconfig.json | 1 + index-cosmetics.ts | 1 - index.ts | 5 +- package-lock.json | 43 +++ package.json | 17 +- rollup.config.js | 28 +- src/content/circumvention.ts | 13 + src/content/circumvention/generic.ts | 55 ++++ src/content/circumvention/instart.ts | 426 +++++++++++++++++++++++++++ src/content/helpers.ts | 35 +++ src/content/injection.ts | 92 ++++++ src/cosmetics-injection.ts | 95 ++---- src/fetch.ts | 82 +++++- src/parsing/network-filter.ts | 6 + tsconfig.json | 6 +- 25 files changed, 877 insertions(+), 178 deletions(-) delete mode 100644 example/rollup.background.js create mode 100644 example/rollup.config.js delete mode 100644 example/rollup.content.js create mode 100644 src/content/circumvention.ts create mode 100644 src/content/circumvention/generic.ts create mode 100644 src/content/circumvention/instart.ts create mode 100644 src/content/helpers.ts create mode 100644 src/content/injection.ts diff --git a/.travis.yml b/.travis.yml index bde2ff9554..0f81d9fcb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,12 +21,6 @@ install: script: - npm test -before_deploy: - - npm run clean - - npm run build - - npm run bundle - - npm run minify - deploy: provider: npm email: remi@cliqz.com diff --git a/CHANGELOG.md b/CHANGELOG.md index 69c3995fc4..3f80760cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Adblocker +## Next + +*not released* + + * Add hard-coded circumvention logic (+ IL defuser) [#59](https://github.com/cliqz-oss/adblocker/pull/59) + - Simplify 'example' extension + - Add circumvention module and entry-point in cosmetics injection + - Clean-up cjs and esm bundles + - Remove obsolete logic to override user-agent in content-script + - Simplify travis config (using new pre* hooks) + - Consolidate 'fetch' module (with metadata about lists) + + ## 0.3.0 *2018-11-20* diff --git a/example/Makefile b/example/Makefile index 9f9a5a469d..e81984d736 100644 --- a/example/Makefile +++ b/example/Makefile @@ -1,22 +1,13 @@ -.PHONY: content-script.bunble.js background.bundle.js build/example watch clean +.PHONY: all watch -ROLLUP="../node_modules/.bin/rollup" - -all: content-script.bundle.js background.bundle.js - -build/example: - tsc -p tsconfig.json --module ES6 --outDir build - -content-script.bundle.js: build/example - ${ROLLUP} -c rollup.content.js - -background.bundle.js: build/example - ${ROLLUP} -c rollup.background.js +all: + npx tsc -p . + npx rollup -c rollup.config.js watch: - concurrently 'tsc -p tsconfig.json --module ES6 --outDir build --watch' 'rollup -c rollup.content.js --watch' 'rollup -c rollup.background.js --watch' + npx concurrently 'tsc -p . --watch' 'rollup -c rollup.config.js --watch' clean: - rm -frv *.bundle.js + rm -frv *.iife.js rm -frv build/ diff --git a/example/README.md b/example/README.md index ee7fa7b05b..402ede7d74 100644 --- a/example/README.md +++ b/example/README.md @@ -1,10 +1,12 @@ # Minimal Content Blocker -This is a minimal webextension doing adblocking/antitracking using most popular -block-lists. It is meant as a very thin wrapper around the [adblocker](https://github.com/cliqz-oss/adblocker) -library. +This is a minimal webextension doing adblocking/antitracking using most +popular block-lists. It is meant as a very thin wrapper around the +[adblocker](https://github.com/cliqz-oss/adblocker) library. You can use it to +debut the adblocker, or test new features. -## Building +## Workflow -1. Build the extension: `make all` -2. Load it in Firefox or Chromium by using the "Load unpacked extension" feature +1. `make watch` will build and monitor for changes +2. Open your browser and load "unpacked extension" from the `example` folder +3. On re-build, reload the extension in browser diff --git a/example/background.ts b/example/background.ts index f50dea9e97..4f8d5ce46b 100644 --- a/example/background.ts +++ b/example/background.ts @@ -28,7 +28,7 @@ function loadAdblocker() { } engine.onUpdateResource([{ filters: resources, checksum: '' }]); - engine.onUpdateFilters(lists, new Set()); + engine.onUpdateFilters(lists, new Set(), true); return engine; }, @@ -68,12 +68,14 @@ chrome.tabs.onCreated.addListener((tab) => { }); chrome.tabs.onUpdated.addListener((_0, _1, tab) => { - if (tabs.has(tab.id)) { - const { source } = tabs.get(tab.id); - if (source !== tab.url) { - resetState(tab.id, tab.url); - updateBadgeCount(tab.id); - } + if (!tabs.has(tab.id)) { + resetState(tab.id, tab.url); + updateBadgeCount(tab.id); + } + const { source } = tabs.get(tab.id); + if (source !== tab.url) { + resetState(tab.id, tab.url); + updateBadgeCount(tab.id); } }); diff --git a/example/content-script.ts b/example/content-script.ts index 530ad3b018..14202bec57 100644 --- a/example/content-script.ts +++ b/example/content-script.ts @@ -17,15 +17,18 @@ import { CosmeticsInjection } from '../index-cosmetics'; */ const backgroundAction = (action, ...args): Promise => { return new Promise((resolve) => { - chrome.runtime.sendMessage({ - action, - args, - }, (response) => { - if (response !== undefined) { - injection.handleResponseFromBackground(response); - } - resolve(); - }); + chrome.runtime.sendMessage( + { + action, + args, + }, + (response) => { + if (response !== undefined) { + injection.handleResponseFromBackground(response); + } + resolve(); + }, + ); }); }; @@ -39,10 +42,9 @@ const backgroundAction = (action, ...args): Promise => { * - Observe mutations in the page (using MutationObserver) and inject relevant * filters for new nodes of the DOM. */ -const injection = new CosmeticsInjection( - window, - backgroundAction, -); +const injection = new CosmeticsInjection(window, backgroundAction); + +injection.injectCircumvention(); /** * Make sure we clean-up all resources and event listeners when this content diff --git a/example/manifest.json b/example/manifest.json index 2618eb2904..1b600dd725 100644 --- a/example/manifest.json +++ b/example/manifest.json @@ -16,7 +16,7 @@ ], "background": { "scripts": [ - "background.bundle.js" + "background.iife.js" ] }, "browser_action": { @@ -27,7 +27,7 @@ "match_about_blank": true, "all_frames": true, "js": [ - "content-script.bundle.js" + "content-script.iife.js" ], "matches": [ "http://*/*", diff --git a/example/rollup.background.js b/example/rollup.background.js deleted file mode 100644 index 45ec265b9d..0000000000 --- a/example/rollup.background.js +++ /dev/null @@ -1,10 +0,0 @@ -import configs from '../rollup.config.js'; - -export default { - input: './build/example/background.js', - output: { - file: 'background.bundle.js', - format: 'iife', - }, - plugins: configs[0].plugins, -}; diff --git a/example/rollup.config.js b/example/rollup.config.js new file mode 100644 index 0000000000..e275fd1203 --- /dev/null +++ b/example/rollup.config.js @@ -0,0 +1,27 @@ +import resolve from 'rollup-plugin-node-resolve'; +import commonjs from 'rollup-plugin-commonjs'; + + +const plugins = [ + resolve(), + commonjs(), +]; + +export default [ + { + input: './build/example/background.js', + output: { + file: 'background.iife.js', + format: 'iife', + }, + plugins, + }, + { + input: './build/example/content-script.js', + output: { + file: 'content-script.iife.js', + format: 'iife', + }, + plugins, + }, +]; diff --git a/example/rollup.content.js b/example/rollup.content.js deleted file mode 100644 index 8249baadbb..0000000000 --- a/example/rollup.content.js +++ /dev/null @@ -1,11 +0,0 @@ -import configs from '../rollup.config.js'; - - -export default { - input: './build/example/content-script.js', - output: { - file: 'content-script.bundle.js', - format: 'iife', - }, - plugins: configs[0].plugins, -}; diff --git a/example/tsconfig.json b/example/tsconfig.json index 05cb4c0e16..8c0ee2ebda 100644 --- a/example/tsconfig.json +++ b/example/tsconfig.json @@ -1,6 +1,7 @@ { "extends": "../tsconfig.json", "compilerOptions": { + "outDir": "build", "noImplicitAny": false }, "files": [ diff --git a/index-cosmetics.ts b/index-cosmetics.ts index a7b31d3162..c30b742aaf 100644 --- a/index-cosmetics.ts +++ b/index-cosmetics.ts @@ -1,2 +1 @@ export { default as CosmeticsInjection } from './src/cosmetics-injection'; -export { overrideUserAgent } from './src/cosmetics-injection'; diff --git a/index.ts b/index.ts index 7d6c26a4e4..6a78d0600f 100644 --- a/index.ts +++ b/index.ts @@ -1,6 +1,5 @@ // Cosmetic injection export { default as CosmeticsInjection } from './src/cosmetics-injection'; -export { overrideUserAgent } from './src/cosmetics-injection'; // Blocking export { default as FiltersEngine } from './src/engine/engine'; @@ -8,8 +7,8 @@ export { default as ReverseIndex } from './src/engine/reverse-index'; export { default as Request } from './src/request'; export { deserializeEngine } from './src/serialization'; -export {default as matchCosmeticFilter } from './src/matching/cosmetics'; -export {default as matchNetworkFilter } from './src/matching/network'; +export { default as matchCosmeticFilter } from './src/matching/cosmetics'; +export { default as matchNetworkFilter } from './src/matching/network'; export { parseCosmeticFilter } from './src/parsing/cosmetic-filter'; export { parseNetworkFilter } from './src/parsing/network-filter'; diff --git a/package-lock.json b/package-lock.json index 3198821cf9..42b1f17bcf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2028,6 +2028,12 @@ "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=", "dev": true }, + "estree-walker": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.5.2.tgz", + "integrity": "sha512-XpCnW/AE10ws/kDAs37cngSkvgIR8aN3G0MS85m7dUpuK2EREo9VJ00uvw6Dg/hXEpfsE1I1TvJOJr+Z+TL+ig==", + "dev": true + }, "esutils": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", @@ -4647,6 +4653,15 @@ "yallist": "^2.1.2" } }, + "magic-string": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.1.tgz", + "integrity": "sha512-sCuTz6pYom8Rlt4ISPFn6wuFodbKMIHUMv4Qko9P17dpxb7s52KJTmRuZZqHdGmLCK9AOcDare039nRIcfdkEg==", + "dev": true, + "requires": { + "sourcemap-codec": "^1.4.1" + } + }, "make-error": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz", @@ -5752,6 +5767,18 @@ "@types/node": "*" } }, + "rollup-plugin-commonjs": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/rollup-plugin-commonjs/-/rollup-plugin-commonjs-9.2.0.tgz", + "integrity": "sha512-0RM5U4Vd6iHjL6rLvr3lKBwnPsaVml+qxOGaaNUWN1lSq6S33KhITOfHmvxV3z2vy9Mk4t0g4rNlVaJJsNQPWA==", + "dev": true, + "requires": { + "estree-walker": "^0.5.2", + "magic-string": "^0.25.1", + "resolve": "^1.8.1", + "rollup-pluginutils": "^2.3.3" + } + }, "rollup-plugin-node-resolve": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-3.4.0.tgz", @@ -5771,6 +5798,16 @@ } } }, + "rollup-pluginutils": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.3.3.tgz", + "integrity": "sha512-2XZwja7b6P5q4RZ5FhyX1+f46xi1Z3qBKigLRZ6VTZjwbN0K1IFGMlwm06Uu0Emcre2Z63l77nq/pzn+KxIEoA==", + "dev": true, + "requires": { + "estree-walker": "^0.5.2", + "micromatch": "^2.3.11" + } + }, "rsvp": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-3.6.2.tgz", @@ -6376,6 +6413,12 @@ "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=", "dev": true }, + "sourcemap-codec": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.4.tgz", + "integrity": "sha512-CYAPYdBu34781kLHkaW3m6b/uUSyMOC2R61gcYMWooeuaGtjof86ZA/8T+qVPPt7np1085CR9hmMGrySwEc8Xg==", + "dev": true + }, "spawn-command": { "version": "0.0.2-1", "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2-1.tgz", diff --git a/package.json b/package.json index 942b0ce0c2..5afd53fae2 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,9 @@ }, "author": "Cliqz", "license": "MPL-2.0", - "main": "dist/cjs/index.js", - "module": "dist/es6/index.js", - "jsnext:main": "dist/es6/index.js", + "browser": "dist/adblocker.umd.js", + "main": "dist/adblocker.cjs.js", + "module": "dist/adblocker.esm.js", "types": "dist/types/index.d.ts", "files": [ "dist", @@ -20,11 +20,13 @@ "scripts": { "clean": "rm -rfv dist", "lint": "tslint -c tslint.json 'src/**/*.ts'", - "build-es6": "tsc -p tsconfig.json --module ES6 --outDir dist/es6", - "build-cjs": "tsc -p tsconfig.json --module commonjs --outDir dist/cjs", - "build": "npm run build-es6 && npm run build-cjs", + "build": "tsc -p tsconfig.json", "bundle": "rollup -c rollup.config.js", - "minify": "google-closure-compiler --js=./dist/adblocker.umd.js --js_output_file=./dist/adblocker.umd.min.js", + "minify": "google-closure-compiler --js=./dist/adblocker.umd.js --js_output_file=./dist/adblocker.umd.min.js && google-closure-compiler --js=./dist/adblocker-cosmetics.umd.js --js_output_file=./dist/adblocker-cosmetics.umd.min.js", + "prebuild": "npm run clean", + "prebundle": "npm run build", + "preminify": "npm run bundle", + "prepack": "npm run minify", "pretest": "npm run lint", "test": "jest --coverage --no-cache ./test/", "dev": "jest --watch ./test/" @@ -47,6 +49,7 @@ "jest": "^23.6.0", "jsdom": "^11.12.0", "rollup": "^0.67.0", + "rollup-plugin-commonjs": "^9.2.0", "rollup-plugin-node-resolve": "^3.4.0", "ts-jest": "^23.10.4", "tslint": "^5.11.0", diff --git a/rollup.config.js b/rollup.config.js index c3d0072ac6..44877e9cea 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,18 +1,16 @@ import resolve from 'rollup-plugin-node-resolve'; +import commonjs from 'rollup-plugin-commonjs'; +import pkg from './package.json'; const plugins = [ - resolve({ - module: true, - jsnext: true, - main: false, - preferBuiltins: false, - modulesOnly: true, - }), + resolve(), + commonjs(), ]; export default [ + // Custom bundle for content-script, contains only a small subset { - input: './dist/es6/index-cosmetics.js', + input: './build/index-cosmetics.js', output: { file: './dist/adblocker-cosmetics.umd.js', name: 'adblocker', @@ -20,13 +18,23 @@ export default [ }, plugins, }, + // Browser-friendly bundle { - input: './dist/es6/index.js', + input: './build/index.js', output: { - file: './dist/adblocker.umd.js', + file: pkg.browser, name: 'adblocker', format: 'umd', }, plugins, }, + // Commonjs and ES module bundles (without third-party deps) + { + input: './build/index.js', + external: ['tldts', 'tslib'], + output: [ + { file: pkg.module, format: 'es' }, + { file: pkg.main, format: 'cjs' }, + ], + }, ]; diff --git a/src/content/circumvention.ts b/src/content/circumvention.ts new file mode 100644 index 0000000000..7e1075b869 --- /dev/null +++ b/src/content/circumvention.ts @@ -0,0 +1,13 @@ +import instart from './circumvention/instart'; + +/** + * This module exports a single function which will be called from content + * script to inject some circumvention logic without having to perform a + * round-trip to the background. This is required for some websites where + * circumvention is done from the main document and counter-measures need to + * take effect immediately. + */ +export default (window: Window): void => { + // Custom logic to defuse InstartLogic + instart(window); +}; diff --git a/src/content/circumvention/generic.ts b/src/content/circumvention/generic.ts new file mode 100644 index 0000000000..e987d8df2d --- /dev/null +++ b/src/content/circumvention/generic.ts @@ -0,0 +1,55 @@ +import { bundle } from '../injection'; + +/** + * This module exports scriptlets building blocks which can be injected in + * pages. Each scriptlet is a function which can accept some arguments as well + * as a list of dependencies (functions which need to be available in the scope + * whenever the script runs in the page). + */ + +/** + * Intercept and ignore errors originating from one of our scripts (if it + * contains the `magic` string which is unique to each content-script) + */ +export const swallowOwnErrors = bundle((magic) => { + // Keep track of original `onerror` callback, if any + const windowOnError = window.onerror; + + // Wrap `onerror` into our custom handler to intercept our own exceptions + const customOnError = function(this: Window, msg: any) { + if (typeof msg === 'string' && msg.indexOf(magic) !== -1) { + return true; // do not fire default event handler + } + + if (windowOnError instanceof Function) { + return windowOnError.apply(this, arguments); + } + + return false; + }.bind(window); + + Object.defineProperty(window, 'onerror', { + get: () => customOnError, + set: () => { + /* Cannot override */ + }, + }); +}); + +export const protectConsole = bundle(() => { + const originalConsole = window.console; + const originalLog = console.log; + + Object.defineProperty(console, 'log', { + value: function fakeLog() { + for (let i = 0; i < arguments.length; i++) { + if (arguments[i] instanceof HTMLElement) { + return; + } + } + return originalLog.apply(originalConsole, arguments); + }.bind(console), + }); + + Object.defineProperty(console.log, 'name', { value: 'log' }); +}); diff --git a/src/content/circumvention/instart.ts b/src/content/circumvention/instart.ts new file mode 100644 index 0000000000..05b5ec13b2 --- /dev/null +++ b/src/content/circumvention/instart.ts @@ -0,0 +1,426 @@ +import { getWindowHostname, magic as contentScriptMagic } from '../helpers'; +import { bundle } from '../injection'; +import { swallowOwnErrors } from './generic'; + +/** + * This module contains custom logic to defuse InstartLogic. It was heavily + * inspired by work done by uBlockOrigin's and AdGuard's developers. + */ + +function isCurrentScriptInstart(thisScript: HTMLScriptElement | SVGScriptElement | null): boolean { + const script = document.currentScript; + + if (script === thisScript) { + return false; + } + + if (script instanceof HTMLScriptElement) { + const src = script.src; + if (src.indexOf('instart.js') !== -1 || src.indexOf('?:i10c.') !== -1) { + return true; + } + + const scriptContent: string | null = script.textContent; + if ( + scriptContent !== null && + (scriptContent.indexOf(':Instart-') !== -1 || + scriptContent.indexOf('I10C') !== -1 || + scriptContent.indexOf('IXC') !== -1 || + scriptContent.indexOf('INSTART') !== -1) + ) { + return true; + } + } + + return false; +} + +/** + * Protect a few global objects from being accessed or set by a script from IL. + */ +const shieldPropertiesFromInstart = bundle( + (magic) => { + const thisScript = document.currentScript; + [ + 'atob', + 'console.error', + 'INSTART_TARGET_NAME', + 'navigator.userAgent', + 'performance', + 'require', + ].forEach((target: string) => { + // Handle nested targets + let obj: any = window; + const chain = target.split('.'); + const prop = chain.pop(); + chain.forEach((part: string) => { + obj = obj[part]; + }); + + if (prop === undefined || obj === undefined) { + return; + } + + // Protect obj[prop] by preventing access to InstartLogic + Object.defineProperty( + obj, + prop, + (() => { + let value = obj[prop]; + return { + get: () => { + if (isCurrentScriptInstart(thisScript)) { + throw new ReferenceError(magic); + } + return value; + }, + set(a: any) { + if (isCurrentScriptInstart(thisScript)) { + throw new ReferenceError(magic); + } + value = a; + }, + }; + })(), + ); + }); + }, + [isCurrentScriptInstart], +); + +/** + * Mock global objects from IL. + */ +const monkeyPatchNanoVisor = bundle((magic) => { + const HtmlStreamingMock = { + InsertTags(_: any, b: any) { + document.write(b); + }, + MoveTagAndCleanUp() { + /* noop */ + }, + MoveTag() { + /* noop */ + }, + RemoveTags() { + /* noop */ + }, + InterceptNode() { + /* noop */ + }, + PatchBegin() { + /* noop */ + }, + PatchEnd() { + /* noop */ + }, + PatchInit() { + /* noop */ + }, + PatchK() { + /* noop */ + }, + ReloadWithNoHtmlStreaming() { + window.location.reload(true); + }, + RemoveAttributes() { + /* noop */ + }, + UpdateAttributes() { + /* noop */ + }, + SendR() { + /* noop */ + }, + RemoveCurrentScript() { + /* noop */ + }, + }; + + const nanoVisorProxy = new Proxy( + {}, + { + get(target, name) { + switch (name) { + case 'HtmlStreaming': + return HtmlStreamingMock; + default: { + // @ts-ignore + return target[name]; + } + } + }, + set(target, name, value) { + switch (name) { + case 'CanRun': + // @ts-ignore + target.CanRun = () => false; + break; + default: + // @ts-ignore + target[name] = value; + } + return true; + }, + }, + ); + + let instartInit: any; + Object.defineProperty(window, '_bcm_il', { value: true }); + Object.defineProperty(window, 'I10C', { value: nanoVisorProxy }); + Object.defineProperty(window, 'I11C', { value: nanoVisorProxy }); + Object.defineProperty(window, 'INSTART', { + value: new Proxy( + {}, + { + get(target, name) { + switch (name) { + case 'Init': + return (a: any) => { + if ( + a instanceof Object && + typeof a.nanovisorGlobalNameSpace === 'string' && + a.nanovisorGlobalNameSpace !== '' + ) { + // @ts-ignore + window[a.nanovisorGlobalNameSpace] = nanoVisorProxy; + } + + a.disableInjectionXhr = true; + a.enableHtmlStreaming = false; + a.enableQSCallDiffComputationConfig = false; + a.enableQuerySelectorMonitoring = false; + a.partialImage = false; + a.rum = false; + a.serveNanovisorSameDomain = false; + a.useWrapper = false; + a.virtualDomains = 0; + a.virtualizeDomains = []; + + if (instartInit !== undefined) { + instartInit(a); + } + }; + default: + // @ts-ignore + if (target[name] === undefined) { + throw new Error(magic); + } + // @ts-ignore + return target[name]; + } + }, + set(target, name, value) { + switch (name) { + case 'Init': + instartInit = value; + break; + default: + // @ts-ignore + target[name] = value; + } + return true; + }, + }, + ), + }); +}); + +export default function instart(window: Window): void { + // NOTE: this list was partially borrowed from uBlockExtra and AdGuard + if ( + [ + 'afterellen.com', + 'allakhazam.com', + 'americanphotomag.com', + 'atvrider.com', + 'baggersmag.com', + 'baltimoresun.com', + 'boatingmag.com', + 'boston.com', + 'cafemom.com', + 'calgaryherald.com', + 'calgarysun.com', + 'capitalgazette.com', + 'carrollcountytimes.com', + 'cattime.com', + 'cbssports.com', + 'celebslam.com', + 'celebuzz.com', + 'chicagotribune.com', + 'chowhound.com', + 'chron.com', + 'chroniclelive.co.uk', + 'citypaper.com', + 'cnet.com', + 'comingsoon.net', + 'computershopper.com', + 'courant.com', + 'craveonline.com', + 'cruisingworld.com', + 'csgoutpost.com', + 'ctnow.com', + 'cycleworld.com', + 'dailydot.com', + 'dailypress.com', + 'dayzdb.com', + 'deathandtaxesmag.com', + 'delmartimes.net', + 'destinationweddingmag.com', + 'dirtrider.com', + 'diversitybestpractices.com', + 'dogtime.com', + 'dotaoutpost.com', + 'download.cnet.com', + 'edmontonjournal.com', + 'edmontonsun.com', + 'edmunds.com', + 'emedicinehealth.com', + 'esohead.com', + 'everquest.allakhazam.com', + 'everydayhealth.com', + 'extremetech.com', + 'fieldandstream.com', + 'financialpost.com', + 'floridatravellife.com', + 'flyingmag.com', + 'focus.de', + 'gamepedia.com', + 'gamerevolution.com', + 'gamespot.com', + 'geek.com', + 'gofugyourself.com', + 'growthspotter.com', + 'hearthhead.com', + 'hockeysfuture.com', + 'hotbikeweb.com', + 'hoylosangeles.com', + 'ibtimes.com', + 'idigitaltimes.com', + 'ign.com', + 'infinitiev.com', + 'islands.com', + 'lajollalight.com', + 'laptopmag.com', + 'latintimes.com', + 'leaderpost.com', + 'legacy.com', + 'lifewire.com', + 'livescience.com', + 'lolking.net', + 'mamaslatinas.com', + 'marlinmag.com', + 'mcall.com', + 'medicaldaily.com', + 'medicinenet.com', + 'metacritic.com', + 'metrolyrics.com', + 'mmo-champion.com', + 'momtastic.com', + 'montrealgazette.com', + 'motorcyclecruiser.com', + 'motorcyclistonline.com', + 'motortrend.com', + 'msn.com', + 'musicfeeds.com.au', + 'mustangandfords.com', + 'mysanantonio.com', + 'nasdaq.com', + 'nationalpost.com', + 'newsarama.com', + 'newsweek.com', + 'opshead.com', + 'orlandosentinel.com', + 'ottawacitizen.com', + 'ottawasun.com', + 'outdoorlife.com', + 'pcmag.com', + 'playstationlifestyle.net', + 'popphoto.com', + 'popsci.com', + 'ranchosantafereview.com', + 'range365.com', + 'ranker.com', + 'realclearpolitics.com', + 'realitytea.com', + 'redeyechicago.com', + 'salon.com', + 'saltwatersportsman.com', + 'sandiegouniontribune.com', + 'saveur.com', + 'scubadiving.com', + 'scubadivingintro.com', + 'seattlepi.com', + 'sfgate.com', + 'sherdog.com', + 'slate.com', + 'slickdeals.net', + 'southflorida.com', + 'space.com', + 'spin.com', + 'sporcle.com', + 'sportdiver.com', + 'sportfishingmag.com', + 'sportingnews.com', + 'sportrider.com', + 'spox.com', + 'stereogum.com', + 'streetchopperweb.com', + 'sun-sentinel.com', + 'superherohype.com', + 'superstreetbike.com', + 'tenplay.com.au', + 'tf2outpost.com', + 'thebalance.com', + 'thefashionspot.com', + 'thefrisky.com', + 'theprovince.com', + 'thespruce.com', + 'thestarphoenix.com', + 'thesuperficial.com', + 'thoughtcatalog.com', + 'thoughtco.com', + 'timeanddate.com', + 'timesunion.com', + 'tmn.today', + 'tomsguide.com', + 'tomsguide.fr', + 'tomshardware.co.uk', + 'tomshardware.com', + 'tomshardware.de', + 'tomshardware.fr', + 'torontosun.com', + 'totalbeauty.com', + 'trustedreviews.com', + 'tv.com', + 'tvguide.com', + 'tvtropes.org', + 'twincities.com', + 'utvdriver.com', + 'vancouversun.com', + 'vg.no', + 'vibe.com', + 'wakeboardingmag.com', + 'washingtonpost.com', + 'waterskimag.com', + 'webmd.com', + 'wikia.com', + 'windowscentral.com', + 'windsorstar.com', + 'winnipegsun.com', + 'workingmother.com', + 'wowhead.com', + 'wrestlezone.com', + 'xda-developers.com', + 'yachtingmagazine.com', + 'zam.com', + ].indexOf(getWindowHostname(window)) !== -1 + ) { + // Un-comment to debug InstartLogic + // protectConsole(window); + + swallowOwnErrors(window, contentScriptMagic); + monkeyPatchNanoVisor(window, contentScriptMagic); + shieldPropertiesFromInstart(window, contentScriptMagic); + } +} diff --git a/src/content/helpers.ts b/src/content/helpers.ts new file mode 100644 index 0000000000..41ea90bd80 --- /dev/null +++ b/src/content/helpers.ts @@ -0,0 +1,35 @@ +/** + * This module exports a list of helpers which can be used in content script. + * *DO NOT* use these as part of scriptlets as these might not be available in + * the context of pages. + */ + +export function getWindowHostname(window: Window) { + const strip = (hostname: string): string => { + if (hostname.startsWith('www.')) { + return hostname.slice(4); + } + return hostname; + }; + + let win = window; + + while (win) { + const hostname = win.location.hostname; + if (hostname !== '') { + return strip(hostname); + } + + if (win === window.parent) { + break; + } + + win = win.parent; + } + + return ''; +} + +export const magic = Math.abs((Date.now() * 524287) ^ ((Math.random() * 524287) >>> 0)).toString( + 16, +); diff --git a/src/content/injection.ts b/src/content/injection.ts new file mode 100644 index 0000000000..1ed103a364 --- /dev/null +++ b/src/content/injection.ts @@ -0,0 +1,92 @@ +/** + * Wrap a self-executing script into a block of custom logic to remove the + * script tag once execution is terminated. This can be useful to not leave + * traces in the DOM after injections. + */ +function autoRemoveScript(script: string): string { + return ` +try { + ${script} +} catch (ex) { } + +(function() { + var currentScript = document.currentScript; + var parent = currentScript && currentScript.parentNode; + + if (parent) { + parent.removeChild(currentScript); + } +})(); + `; +} + +/** + * Given a self-executing script as well as a list of dependencies (function + * which are required by the injected script), create a script which contains + * both the dependencies (as scoped functions) and the script. + */ +function wrapCallableInContext(script: string, deps: Array<(...args: any[]) => any> = []): string { + return ` +${deps.map((dep) => `const ${dep.name} = ${dep.toString()};`).join('\n')} +${script} + `; +} + +/** + * Given a function which can accept arguments, serialize it into a string (as + * well as its argument) so that it will automatically execute upon injection. + */ +function autoCallFunction(fn: (...args: any[]) => void, ...args: any[]): string { + return ` +try { + (${fn.toString()})(${args.map((arg) => JSON.stringify(arg)).join(', ')}); +} catch (ex) { };`; +} + +export function blockScript(filter: string, doc: Document): void { + const filterRE = new RegExp(filter); + doc.addEventListener('beforescriptexecute', (ev) => { + const target = ev.target as HTMLElement; + if (target.textContent && filterRE.test(target.textContent)) { + ev.preventDefault(); + ev.stopPropagation(); + } + }); +} + +export function injectCSSRule(rule: string, doc: Document): void { + const css = doc.createElement('style'); + css.type = 'text/css'; + css.id = 'cliqz-adblokcer-css-rules'; + const parent = doc.head || doc.documentElement; + if (parent !== null) { + parent.appendChild(css); + css.appendChild(doc.createTextNode(rule)); + } +} + +export function injectScript(s: string, doc: Document): void { + const script = doc.createElement('script'); + script.type = 'text/javascript'; + script.id = 'cliqz-adblocker-script'; + script.async = false; + script.appendChild(doc.createTextNode(autoRemoveScript(s))); + + // Insert node + const parent = doc.head || doc.documentElement; + if (parent !== null) { + parent.appendChild(script); + } +} + +/** + * Given a scriptlet (as well as optional dependencies: symbols which must be + * available in the scope for this scriptlet to do its job), returns a callback + * which needs to be called with the desired window as well as optional + * arguments for the scriptlet. The script will be injected in the head of + * window's document as a self-executing, self-erasing script element. + */ +export function bundle(fn: (...args: any[]) => void, deps: Array<(...args: any[]) => any> = []) { + return (window: Window, ...args: any[]) => + injectScript(wrapCallableInContext(autoCallFunction(fn, ...args), deps), window.document); +} diff --git a/src/cosmetics-injection.ts b/src/cosmetics-injection.ts index e3e65034b2..60f525fcda 100644 --- a/src/cosmetics-injection.ts +++ b/src/cosmetics-injection.ts @@ -1,11 +1,13 @@ +import injectCircumvention from './content/circumvention'; +import { blockScript, injectCSSRule, injectScript } from './content/injection'; // We need this as `MutationObserver` is currently not part of the `Window` type // provided by typescript, although it should be! This will be erased at compile // time so it has no impact on produced code. declare global { - interface Window { - MutationObserver?: typeof MutationObserver; - } + interface Window { + MutationObserver?: typeof MutationObserver; + } } interface IMessageFromBackground { @@ -15,67 +17,6 @@ interface IMessageFromBackground { styles: string[]; } -function injectCSSRule(rule: string, doc: Document): void { - const css = doc.createElement('style'); - css.type = 'text/css'; - css.id = 'cliqz-adblokcer-css-rules'; - const parent = doc.head || doc.documentElement; - if (parent !== null) { - parent.appendChild(css); - css.appendChild(doc.createTextNode(rule)); - } -} - -function injectScript(s: string, doc: Document): void { - // Wrap script so that it removes itself when the execution is over. - const autoRemoveScript = ` - try { - ${s} - } catch (ex) { } - - (function() { - var currentScript = document.currentScript; - var parent = currentScript && currentScript.parentNode; - - if (parent) { - parent.removeChild(currentScript); - } - })(); - `; - - // Create node - const script = doc.createElement('script'); - script.type = 'text/javascript'; - script.id = 'cliqz-adblocker-script'; - script.appendChild(doc.createTextNode(autoRemoveScript)); - - // Insert node - const parent = doc.head || doc.documentElement; - if (parent !== null) { - parent.appendChild(script); - } -} - -function blockScript(filter: string, doc: Document): void { - const filterRE = new RegExp(filter); - doc.addEventListener('beforescriptexecute', ev => { - const target = ev.target as HTMLElement; - if (target.textContent && filterRE.test(target.textContent)) { - ev.preventDefault(); - ev.stopPropagation(); - } - }); -} - -export function overrideUserAgent(): void { - const script = () => { - Object.defineProperty(navigator, 'userAgent', { - get: () => 'Mozilla/5.0 Gecko Firefox', - }); - }; - injectScript(`(${script.toString()})()`, window.document); -} - /** * Takes care of injecting cosmetic filters in a given window. Responsabilities: * - Inject scripts. @@ -105,8 +46,11 @@ export default class CosmeticInjection { private observedNodes: Set; private mutationObserver: MutationObserver | null; - constructor(window: Window, backgroundAction: (action: string, ...args: any[]) => Promise, - useMutationObserver = true) { + constructor( + window: Window, + backgroundAction: (action: string, ...args: any[]) => Promise, + useMutationObserver = true, + ) { this.window = window; this.backgroundAction = backgroundAction; @@ -139,7 +83,16 @@ export default class CosmeticInjection { } } - public handleResponseFromBackground({ active, scripts, blockedScripts, styles }: IMessageFromBackground) { + public injectCircumvention(): void { + injectCircumvention(this.window); + } + + public handleResponseFromBackground({ + active, + scripts, + blockedScripts, + styles, + }: IMessageFromBackground) { if (!active) { this.unload(); return; @@ -204,7 +157,7 @@ export default class CosmeticInjection { * cosmetic filters to inject in the page. */ private onMutation(mutations: Array<{ target: Node }>) { - let targets: Set = new Set(mutations.map(m => m.target).filter(t => t)); + let targets: Set = new Set(mutations.map((m) => m.target).filter((t) => t)); // TODO - it might be necessary to inject scripts, CSS and block scripts // from here into iframes with no src. We could first inject/block @@ -231,7 +184,7 @@ export default class CosmeticInjection { // Collect nodes of targets const nodeInfo = new Set(); - targets.forEach(target => { + targets.forEach((target) => { const nodes = (target as HTMLElement).querySelectorAll('*'); for (let i = 0; i < nodes.length; i += 1) { const node = nodes[i] as HTMLElement; @@ -258,7 +211,7 @@ export default class CosmeticInjection { } if (node.className && node.className.split) { - node.className.split(' ').forEach(name => { + node.className.split(' ').forEach((name) => { const selector = `.${name}`; if (!this.observedNodes.has(selector)) { nodeInfo.add(selector); @@ -278,7 +231,7 @@ export default class CosmeticInjection { private startObserving() { // Attach mutation observer in case the DOM is mutated. if (this.window.MutationObserver !== undefined) { - this.mutationObserver = new this.window.MutationObserver(mutations => + this.mutationObserver = new this.window.MutationObserver((mutations) => this.onMutation(mutations), ); this.mutationObserver.observe(this.window.document, { diff --git a/src/fetch.ts b/src/fetch.ts index 574ddc8977..3560779052 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -2,24 +2,78 @@ function fetchResource(url: string): Promise { return fetch(url).then((response: any) => response.text()); } -const defaultLists = [ - // 'https://easylist-downloads.adblockplus.org/antiadblockfilters.txt', - // 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', - 'https://easylist.to/easylist/easylist.txt', - 'https://easylist.to/easylist/easyprivacy.txt', - 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus&showintro=1&mimetype=plaintext', - 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', - 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', - 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt', - 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt', - 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt', +const enum Category { + Privacy, + Ads, + Unbreak, + Circumvention, + Country, + Misc, +} + +const lists = [ + { + category: Category.Unbreak, + enabledByDefault: true, + url: 'https://easylist-downloads.adblockplus.org/antiadblockfilters.txt', + }, + { + category: Category.Country, + country: 'de', + enabledByDefault: false, + url: 'https://easylist-downloads.adblockplus.org/easylistgermany.txt', + }, + { + category: Category.Ads, + enabledByDefault: true, + url: 'https://easylist.to/easylist/easylist.txt', + }, + { + category: Category.Privacy, + enabledByDefault: false, + url: 'https://easylist.to/easylist/easyprivacy.txt', + }, + { + category: Category.Ads, + enabledByDefault: false, + url: + 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=adblockplus&showintro=1&mimetype=plaintext', + }, + { + category: Category.Misc, + enabledByDefault: false, + url: 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt', + }, + { + category: Category.Ads, + enabledByDefault: true, + url: 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt', + }, + { + category: Category.Privacy, + enabledByDefault: false, + url: 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt', + }, + { + category: Category.Misc, + enabledByDefault: true, + url: + 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt', + }, + { + category: Category.Unbreak, + enabledByDefault: true, + url: 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt', + }, ]; /** - * Fetch latest version of default blocking lists. + * Fetch latest version of enabledByDefault blocking lists. */ -export function fetchLists(lists: string[] = defaultLists): Promise { - return Promise.all(lists.map(fetchResource)); +export function fetchLists(): Promise { + return Promise.all( + lists.filter(({ enabledByDefault }) => enabledByDefault).map(({ url }) => fetchResource(url)), + ); } /** diff --git a/src/parsing/network-filter.ts b/src/parsing/network-filter.ts index b89a7d96e4..d1d612b7c3 100644 --- a/src/parsing/network-filter.ts +++ b/src/parsing/network-filter.ts @@ -732,6 +732,12 @@ export function parseNetworkFilter(rawLine: string): NetworkFilter | null { break; } + case 'badfilter': + // TODO - how to handle those, if we start in mask, then the id will + // differ from the other filter. We could keep original line. How do + // to eliminate thos efficiently? They will probably endup in the same + // bucket, so maybe we could do that on a per-bucket basis? + return null; case 'important': // Note: `negation` should always be `false` here. if (negation) { diff --git a/tsconfig.json b/tsconfig.json index ca651e5c36..7fbc4fdd79 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,9 +2,11 @@ "compilerOptions": { "declaration": true, "declarationDir": "dist/types", - "types": ["jest", "node", "chrome"], + "types": ["jest", "node", "chrome", "jsdom"], "target": "ES3", - "lib": ["ES6", "DOM"], + "lib": ["es2015", "ES6", "DOM"], + "module": "ES6", + "outDir": "build", "newLine": "lf", "removeComments": true, "moduleResolution": "Node",