Fix cosmetic matching (tokenization bug)

ghostery · Dec 3, 2018 · fe366c9 · fe366c9
1 parent 7167736
commit fe366c9
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,12 +4,14 @@
 
 *not released yet*
 
+  * Fix cosmetic matching (tokenization bug) [#65](https://github.com/cliqz-oss/adblocker/pull/65)
+  * Optimize serialization and properly handle unicode in filters [#61](https://github.com/cliqz-oss/adblocker/pull/61)
+
 ## 0.3.1
 
 *2018-11-29*
 
-  * Optimize serialization and properly handle unicode in filters [#61](https://github.com/cliqz-oss/adblocker/pull/61)
-  * Fix fuzzy matching by allowing tokens of any size [#61](https://github.com/cliqz-oss/adblocker/pull/62)
+  * Fix fuzzy matching by allowing tokens of any size [#62](https://github.com/cliqz-oss/adblocker/pull/62)
   * Add support for CSP (Content Security Policy) filters [#60](https://github.com/cliqz-oss/adblocker/pull/60)
   * Add hard-coded circumvention logic (+ IL defuser) [#59](https://github.com/cliqz-oss/adblocker/pull/59)
     - Simplify 'example' extension

diff --git a/src/utils.ts b/src/utils.ts
@@ -160,7 +160,7 @@ function fastTokenizerNoRegex(
     tokensBufferIndex += 1;
   }
 
-  return TOKENS_BUFFER.subarray(0, tokensBufferIndex);
+  return TOKENS_BUFFER.slice(0, tokensBufferIndex);
 }
 
 function fastTokenizer(pattern: string, isAllowedCode: (ch: number) => boolean): Uint32Array {
@@ -187,7 +187,7 @@ function fastTokenizer(pattern: string, isAllowedCode: (ch: number) => boolean):
     tokensBufferIndex += 1;
   }
 
-  return TOKENS_BUFFER.subarray(0, tokensBufferIndex);
+  return TOKENS_BUFFER.slice(0, tokensBufferIndex);
 }
 
 export function tokenize(pattern: string): Uint32Array {