Merge remote-tracking branch 'origin/main' into psalas/tts-buffer

fixie-ai · Nov 18, 2023 · 3a222ab · 3a222ab
2 parents 3b84ffe + 6f27bf8
commit 3a222ab
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 6 deletions.
diff --git a/packages/ai-jsx/package.json b/packages/ai-jsx/package.json
@@ -385,7 +385,7 @@
     "cli-highlight": "^2.1.11",
     "cli-spinners": "^2.9.0",
     "ink": "^4.2.0",
-    "js-tiktoken": "^1.0.7",
+    "js-tiktoken": "^1.0.8",
     "js-yaml": "^4.1.0",
     "langchain": "^0.0.143",
     "lodash": "^4.17.21",

diff --git a/packages/ai-jsx/src/lib/openai.tsx b/packages/ai-jsx/src/lib/openai.tsx
@@ -22,8 +22,7 @@ import { OpenAI as OpenAIClient } from 'openai';
 export { OpenAI as OpenAIClient } from 'openai';
 import { FinalRequestOptions } from 'openai/core';
 import { debugRepresentation } from '../core/debug.js';
-import cl100k_base from 'js-tiktoken/ranks/cl100k_base';
-import { Tiktoken } from 'js-tiktoken/lite';
+import { getEncoding } from 'js-tiktoken';
 import _ from 'lodash';
 
 // https://platform.openai.com/docs/models/model-endpoint-compatibility
@@ -136,7 +135,7 @@ export function OpenAI({
 }
 
 // Preload the tokenizer to avoid a large delay on first use.
-const cl100kTokenizer = new Tiktoken(cl100k_base);
+const cl100kTokenizer = getEncoding('cl100k_base');
 export const tokenizer = {
   encode: (text: string) => cl100kTokenizer.encode(text),
   decode: (tokens: number[]) => cl100kTokenizer.decode(tokens),

diff --git a/packages/docs/docs/changelog.md b/packages/docs/docs/changelog.md
@@ -1,6 +1,10 @@
 # Changelog
 
-## 0.26.0
+## 0.26.1
+
+- Fix `js-tiktoken` import that fails on 1.0.8.
+
+## [0.26.0](https://github.com/fixie-ai/ai-jsx/commit/c5501a1d1016a3cb27259a79bc38b8d68942f761)
 
 - In the `Sidekick` component:
   - Remove the MDX repair attempt.

diff --git a/yarn.lock b/yarn.lock
@@ -7306,7 +7306,7 @@ __metadata:
     globby: ^13.1.4
     ink: ^4.2.0
     jest: ^29.5.0
-    js-tiktoken: ^1.0.7
+    js-tiktoken: ^1.0.8
     js-yaml: ^4.1.0
     langchain: ^0.0.143
     load-json-file: ^7.0.1
@@ -16413,6 +16413,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"js-tiktoken@npm:^1.0.8":
+  version: 1.0.8
+  resolution: "js-tiktoken@npm:1.0.8"
+  dependencies:
+    base64-js: ^1.5.1
+  checksum: ac6e666f14661b4e744bd60987e35275668863a686413bb35baca2d9a503bc95fcfa907f3c02b8e8abe4a4a70abe622c06cbb72ce3574b3b929d22f9146c3f85
+  languageName: node
+  linkType: hard
+
 "js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0":
   version: 4.0.0
   resolution: "js-tokens@npm:4.0.0"