Tldts experimental

.travis.yml

@@ -14,7 +14,7 @@ before_install:
 cache: yarn
 
 script:
-    - travis_wait 20000 ./run_tests.sh
+    - ./run_tests.sh
 
 # TODO - migrate to lerna
 # deploy:

CHANGELOG.md

@@ -4,6 +4,11 @@
 
 *not released yet*
 
+  * switch from `tldts` to `tldts-experimental` package [#183](https://github.com/cliqz-oss/adblocker/pull/183)
+    * add `bootstrap` to root package.json
+    * switch from `tldts` to `tldts-experimental` (faster, smaller bundles)
+    * switch internal commands from `npm` to `yarn`
+    * remove redundant `lint` during CI
   * change structure of the cliqz/adblocker project into a monorepo [#181](https://github.com/cliqz-oss/adblocker/pull/181)
     * embrace `lerna` and `yarn` workspaces as a way to manage multiple packages
     * adopt conventional commits as a way to structure contributions

bench/dataset/create_list.js

@@ -2,8 +2,8 @@ const fs = require('fs');
 const path = require('path');
 const readline = require('readline');
 
-const adblocker = require(path.resolve(__dirname, '../../dist/adblocker.umd.min.js'));
-const { getPublicSuffix, getDomain } = require('tldts');
+const adblocker = require('@cliqz/adblocker');
+const { getPublicSuffix, getDomain } = require('tldts-experimental');
 const top50kDomains = require('./hostnames');
 
 function getHostnameWithoutSuffix(hostname) {

lerna-lint.js

@@ -199,7 +199,7 @@ const EXPECTED_FIELDS = new Set([
       });
       assertMetadata(
         'scripts.build',
-        "concurrently 'npm run build-cjs' 'npm run build-es6'",
+        "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
         package,
         { optional: true },
       );
@@ -248,28 +248,28 @@ const EXPECTED_FIELDS = new Set([
 
     assertMetadata(
       'scripts.prepack',
-      'npm run bundle',
+      'yarn run bundle',
       package,
       { optional: true },
     );
 
     assertMetadata(
       'scripts.prebundle',
-      'npm run build',
+      'yarn run build',
       package,
       { optional: true },
     );
 
     assertMetadata(
       'scripts.prebuild',
-      'npm run clean',
+      'yarn run clean',
       package,
       { optional: true },
     );
 
     assertMetadata(
       'scripts.pretest',
-      'npm run lint',
+      'yarn run lint',
       package,
       { optional: true },
     );

package.json

@@ -16,15 +16,16 @@
   "husky": {
     "hooks": {
       "commit-msg": "commitlint -E HUSKY_GIT_PARAMS",
-      "pre-push": "npm run lint"
+      "pre-push": "yarn lint"
     }
   },
   "scripts": {
+    "bootstrap": "npx lerna bootstrap",
     "lint": "lerna run --parallel lint",
     "bundle": "lerna run --concurrency 1 bundle",
     "test": "lerna run --concurrency 1 test",
     "clean": "lerna run --parallel clean && lerna clean --yes && rimraf node_modules",
-    "prepublish": "npm run lint"
+    "prepublish": "yarn lint"
   },
   "devDependencies": {
     "@commitlint/cli": "^8.0.0",

packages/adblocker-benchmarks/blockers/duckduckgo.js

@@ -7,7 +7,7 @@
  */
 
 const abp = require('abp-filter-parser');
-const { getHostname } = require('tldts');
+const { getHostname } = require('tldts-experimental');
 
 // This maps webRequest types to DuckDuckGo types
 const DDG_OPTIONS = {

packages/adblocker-benchmarks/blockers/tldts_baseline.js

@@ -6,7 +6,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-const { parse } = require('tldts');
+const { parse } = require('tldts-experimental');
 
 module.exports = class Tldts {
   static parse() {

packages/adblocker-benchmarks/create_dataset.js

@@ -11,7 +11,7 @@
 const fs = require('fs');
 const createPuppeteerPool = require('puppeteer-pool');
 const stream = require('stream');
-const { getDomain } = require('tldts');
+const { getDomain } = require('tldts-experimental');
 
 class RequestStreamer extends stream.Readable {
   constructor(options) {

packages/adblocker-benchmarks/package.json

@@ -14,7 +14,7 @@
     "download-requests": "make requests.json",
     "download-abp": "make adblockpluscore",
     "download-brave": "make adblock-rs",
-    "prepare": "npm run download-requests && npm run download-abp && npm run download-brave"
+    "prepare": "yarn download-requests && yarn download-abp && yarn download-brave"
   },
   "bugs": {
     "url": "https://github.com/cliqz-oss/adblocker/issues"

packages/adblocker-circumvention/package.json

@@ -26,11 +26,11 @@
     "lint": "tslint --config ../../tslint.json --project .",
     "build-cjs": "tsc -p . --outDir build/cjs --module commonjs",
     "build-es6": "tsc -p . --outDir build/es6 --module esnext",
-    "build": "concurrently 'npm run build-cjs' 'npm run build-es6'",
-    "prebuild": "npm run clean",
+    "build": "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
+    "prebuild": "yarn run clean",
     "bundle": "rollup --config rollup.config.ts",
-    "prebundle": "npm run build",
-    "prepack": "npm run bundle"
+    "prebundle": "yarn run build",
+    "prepack": "yarn run bundle"
   },
   "bugs": {
     "url": "https://github.com/cliqz-oss/adblocker/issues"

packages/adblocker-electron/package.json

@@ -26,13 +26,12 @@
     "lint": "tslint --config ../../tslint.json --project .",
     "build-cjs": "tsc -p . --outDir build/cjs --module commonjs",
     "build-es6": "tsc -p . --outDir build/es6 --module esnext",
-    "build": "concurrently 'npm run build-cjs' 'npm run build-es6'",
-    "prebuild": "npm run clean",
+    "build": "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
+    "prebuild": "yarn run clean",
     "bundle": "rollup --config rollup.config.ts",
-    "prebundle": "npm run build",
-    "prepack": "npm run bundle",
-    "test": "jest --config ./jest.config.js --coverage --no-cache ./adblocker.test.ts",
-    "pretest": "npm run lint"
+    "prebundle": "yarn run build",
+    "prepack": "yarn run bundle",
+    "test": "jest --config ./jest.config.js --coverage ./adblocker.test.ts"
   },
   "peerDependencies": {
     "electron": "^5.0.6"

packages/adblocker-puppeteer-example/README.md

@@ -1,5 +1,5 @@
 Blocking ads in puppeteer using @cliqz/adblocker.
 
 ```sh
-$ npm run start
+$ yarn start
 ```

packages/adblocker-puppeteer/adblocker.ts

@@ -8,7 +8,7 @@
 
 // Type definition
 import * as puppeteer from 'puppeteer';
-import { parse } from 'tldts';
+import { parse } from 'tldts-experimental';
 
 import {
   autoRemoveScript,

packages/adblocker-puppeteer/package.json

@@ -25,13 +25,12 @@
     "lint": "tslint --config ../../tslint.json --project .",
     "build-cjs": "tsc -p . --outDir build/cjs --module commonjs",
     "build-es6": "tsc -p . --outDir build/es6 --module esnext",
-    "build": "concurrently 'npm run build-cjs' 'npm run build-es6'",
-    "prebuild": "npm run clean",
+    "build": "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
+    "prebuild": "yarn run clean",
     "bundle": "rollup --config rollup.config.ts",
-    "prebundle": "npm run build",
-    "prepack": "npm run bundle",
-    "test": "jest --config ./jest.config.js --coverage --no-cache ./adblocker.test.ts",
-    "pretest": "npm run lint"
+    "prebundle": "yarn run build",
+    "prepack": "yarn run bundle",
+    "test": "jest --config ./jest.config.js --coverage ./adblocker.test.ts"
   },
   "bugs": {
     "url": "https://github.com/cliqz-oss/adblocker/issues"
@@ -42,7 +41,7 @@
   "dependencies": {
     "@cliqz/adblocker": "^0.12.1-alpha.3",
     "@types/puppeteer": "^1.12.4",
-    "tldts": "^5.2.1"
+    "tldts-experimental": "^5.3.0"
   },
   "devDependencies": {
     "@ampproject/rollup-plugin-closure-compiler": "^0.9.0",

packages/adblocker-webextension-cosmetics/package.json

@@ -26,13 +26,12 @@
     "lint": "tslint --config ../../tslint.json --project .",
     "build-cjs": "tsc -p . --outDir build/cjs --module commonjs",
     "build-es6": "tsc -p . --outDir build/es6 --module esnext",
-    "build": "concurrently 'npm run build-cjs' 'npm run build-es6'",
-    "prebuild": "npm run clean",
+    "build": "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
+    "prebuild": "yarn run clean",
     "bundle": "rollup --config rollup.config.ts",
-    "prebundle": "npm run build",
-    "prepack": "npm run bundle",
-    "test": "jest --config ./jest.config.js --coverage --no-cache ./adblocker.test.ts",
-    "pretest": "npm run lint"
+    "prebundle": "yarn run build",
+    "prepack": "yarn run bundle",
+    "test": "jest --config ./jest.config.js --coverage ./adblocker.test.ts"
   },
   "bugs": {
     "url": "https://github.com/cliqz-oss/adblocker/issues"

packages/adblocker-webextension-example/README.md

@@ -7,6 +7,6 @@ debug the adblocker, or test new features.
 
 ## Workflow
 
-1. `npm run bundle` will build the extension
+1. `yarn bundle` will build the extension
 2. Open your browser and load "unpacked extension" from the `example` folder
 3. On re-build, reload the extension in browser

packages/adblocker-webextension-example/package.json

@@ -21,11 +21,11 @@
     "lint": "tslint --config ../../tslint.json --project .",
     "build-cjs": "tsc -p . --outDir build/cjs --module commonjs",
     "build-es6": "tsc -p . --outDir build/es6 --module esnext",
-    "build": "concurrently 'npm run build-cjs' 'npm run build-es6'",
-    "prebuild": "npm run clean",
+    "build": "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
+    "prebuild": "yarn run clean",
     "bundle": "rollup --config rollup.config.ts",
-    "prebundle": "npm run build",
-    "prepack": "npm run bundle"
+    "prebundle": "yarn run build",
+    "prepack": "yarn run bundle"
   },
   "bugs": {
     "url": "https://github.com/cliqz-oss/adblocker/issues"

packages/adblocker-webextension/adblocker.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { parse } from 'tldts';
+import { parse } from 'tldts-experimental';
 
 import { FiltersEngine, Request, WebRequestType } from '@cliqz/adblocker';
 import {

packages/adblocker-webextension/package.json

@@ -23,13 +23,12 @@
     "lint": "tslint --config ../../tslint.json --project .",
     "build-cjs": "tsc -p . --outDir build/cjs --module commonjs",
     "build-es6": "tsc -p . --outDir build/es6 --module esnext",
-    "build": "concurrently 'npm run build-cjs' 'npm run build-es6'",
-    "prebuild": "npm run clean",
+    "build": "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
+    "prebuild": "yarn run clean",
     "bundle": "rollup --config rollup.config.ts",
-    "prebundle": "npm run build",
-    "prepack": "npm run bundle",
-    "test": "jest --config ./jest.config.js --coverage --no-cache ./adblocker.test.ts",
-    "pretest": "npm run lint"
+    "prebundle": "yarn run build",
+    "prepack": "yarn run bundle",
+    "test": "jest --config ./jest.config.js --coverage ./adblocker.test.ts"
   },
   "publishConfig": {
     "access": "public"
@@ -52,6 +51,6 @@
   "dependencies": {
     "@cliqz/adblocker": "^0.12.1-alpha.3",
     "@cliqz/adblocker-webextension-cosmetics": "^0.12.1-alpha.3",
-    "tldts": "^5.2.1"
+    "tldts-experimental": "^5.3.0"
   }
 }

packages/adblocker/package.json

@@ -26,13 +26,12 @@
     "lint": "tslint --config ../../tslint.json --project .",
     "build-cjs": "tsc -p . --outDir build/cjs --module commonjs",
     "build-es6": "tsc -p . --outDir build/es6 --module esnext",
-    "build": "concurrently 'npm run build-cjs' 'npm run build-es6'",
-    "prebuild": "npm run clean",
+    "build": "concurrently 'yarn run build-cjs' 'yarn run build-es6'",
+    "prebuild": "yarn run clean",
     "bundle": "rollup --config rollup.config.ts",
-    "prebundle": "npm run build",
-    "prepack": "npm run bundle",
-    "test": "jest --config ./jest.config.js --coverage --no-cache ./test/",
-    "pretest": "npm run lint",
+    "prebundle": "yarn run build",
+    "prepack": "yarn run bundle",
+    "test": "jest --config ./jest.config.js --coverage ./test/",
     "generate-codebooks": "ts-node ./tools/generate_compression_codebooks.ts"
   },
   "bugs": {
@@ -53,7 +52,7 @@
     "tslint-config-prettier": "^1.18.0"
   },
   "dependencies": {
-    "tldts": "^5.2.1",
+    "tldts-experimental": "^5.3.0",
     "tslib": "^1.10.0",
     "tsmaz": "^1.2.2"
   }

packages/adblocker/src/request.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { parse } from 'tldts';
+import { parse } from 'tldts-experimental';
 import TokensBuffer from './tokens-buffer';
 import { createFuzzySignature, fastHash, tokenizeInPlace } from './utils';
 

packages/adblocker/test/engine.test.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { getDomain } from 'tldts';
+import { getDomain } from 'tldts-experimental';
 
 import Engine from '../src/engine/engine';
 import NetworkFilter from '../src/filters/network';

packages/adblocker/test/matching.test.ts

@@ -6,7 +6,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-import { getDomain } from 'tldts';
+import { getDomain } from 'tldts-experimental';
 
 import CosmeticFilter, {
   getHashesFromLabelsBackward,

run_tests.sh

@@ -5,8 +5,9 @@ set -o nounset
 set -o xtrace
 set -o pipefail
 
-npx lerna bootstrap
+yarn bootstrap
 node licenser.js --ci
 node lerna-lint.js --ci
-npm run bundle
-npm run test
+yarn lint
+yarn bundle
+yarn test

yarn.lock

@@ -1358,14 +1358,14 @@
     parse5 "^4.0.0"
 
 "@types/node@*", "@types/node@^12.0.10", "@types/node@^12.0.3":
-  version "12.0.10"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.0.10.tgz#51babf9c7deadd5343620055fc8aff7995c8b031"
-  integrity sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==
+  version "12.0.12"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.0.12.tgz#cc791b402360db1eaf7176479072f91ee6c6c7ca"
+  integrity sha512-Uy0PN4R5vgBUXFoJrKryf5aTk3kJ8Rv3PdlHjl6UaX+Cqp1QE0yPQ68MPXGrZOfG7gZVNDIJZYyot0B9ubXUrQ==
 
 "@types/node@^10.12.18":
-  version "10.14.10"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.10.tgz#e491484c6060af8d461e12ec81c0da8a3282b8de"
-  integrity sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q==
+  version "10.14.12"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.12.tgz#0eec3155a46e6c4db1f27c3e588a205f767d622f"
+  integrity sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==
 
 "@types/normalize-package-data@^2.4.0":
   version "2.4.0"
@@ -7201,10 +7201,17 @@ through@2, "through@>=2.2.7 <3", through@^2.3.4, through@^2.3.6:
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
   integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
 
-tldts@^5.2.1:
-  version "5.2.1"
-  resolved "https://registry.yarnpkg.com/tldts/-/tldts-5.2.1.tgz#8fe745feb8b822deda8d3677cf836a77028056db"
-  integrity sha512-/K8BtLl2ZQUmd3E5ecwlf73+7fPIBKBQpGg/6cQtclobqyDy6bQ8shgXwHiw9SDUmaBAQvJe832o4MFNeWk29g==
+tldts-core@^5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-5.3.0.tgz#22c817ce33273134fc2105c523bc86fb1bac2c2d"
+  integrity sha512-EGEgyBPavRj6i3EalLtFLbrpjlHKy1WFsCHmeViNj0XEAU1RkueKbbtZ+n7dhMqB4sjwCBiqpUqdLZAGa+pYCw==
+
+tldts-experimental@^5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/tldts-experimental/-/tldts-experimental-5.3.0.tgz#f3d6231899de83093c236ba6f11efdb933ab67b4"
+  integrity sha512-jLlMvwe7FNmZl/S4Y+dvxbDkKoZscyd+TYT8r+gcn4GsigUI9CwJLfsTyu4+PG8DwxHpWskPf+p8PvXeI0g30Q==
+  dependencies:
+    tldts-core "^5.3.0"
 
 tmp@^0.0.33:
   version "0.0.33"