diff --git a/packages/ai-jsx/package.json b/packages/ai-jsx/package.json index e41ebce8..02beb7c0 100644 --- a/packages/ai-jsx/package.json +++ b/packages/ai-jsx/package.json @@ -385,7 +385,7 @@ "cli-highlight": "^2.1.11", "cli-spinners": "^2.9.0", "ink": "^4.2.0", - "js-tiktoken": "^1.0.7", + "js-tiktoken": "^1.0.8", "js-yaml": "^4.1.0", "langchain": "^0.0.143", "lodash": "^4.17.21", diff --git a/packages/ai-jsx/src/lib/openai.tsx b/packages/ai-jsx/src/lib/openai.tsx index 076a68d3..4e69624e 100644 --- a/packages/ai-jsx/src/lib/openai.tsx +++ b/packages/ai-jsx/src/lib/openai.tsx @@ -22,8 +22,7 @@ import { OpenAI as OpenAIClient } from 'openai'; export { OpenAI as OpenAIClient } from 'openai'; import { FinalRequestOptions } from 'openai/core'; import { debugRepresentation } from '../core/debug.js'; -import cl100k_base from 'js-tiktoken/ranks/cl100k_base'; -import { Tiktoken } from 'js-tiktoken/lite'; +import { getEncoding } from 'js-tiktoken'; import _ from 'lodash'; // https://platform.openai.com/docs/models/model-endpoint-compatibility @@ -136,7 +135,7 @@ export function OpenAI({ } // Preload the tokenizer to avoid a large delay on first use. -const cl100kTokenizer = new Tiktoken(cl100k_base); +const cl100kTokenizer = getEncoding('cl100k_base'); export const tokenizer = { encode: (text: string) => cl100kTokenizer.encode(text), decode: (tokens: number[]) => cl100kTokenizer.decode(tokens), diff --git a/packages/docs/docs/changelog.md b/packages/docs/docs/changelog.md index 8abae386..5819527c 100644 --- a/packages/docs/docs/changelog.md +++ b/packages/docs/docs/changelog.md @@ -1,6 +1,10 @@ # Changelog -## 0.26.0 +## 0.26.1 + +- Fix `js-tiktoken` import that fails on 1.0.8. + +## [0.26.0](https://github.com/fixie-ai/ai-jsx/commit/c5501a1d1016a3cb27259a79bc38b8d68942f761) - In the `Sidekick` component: - Remove the MDX repair attempt. diff --git a/yarn.lock b/yarn.lock index 9c75ac7f..e324065c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7306,7 +7306,7 @@ __metadata: globby: ^13.1.4 ink: ^4.2.0 jest: ^29.5.0 - js-tiktoken: ^1.0.7 + js-tiktoken: ^1.0.8 js-yaml: ^4.1.0 langchain: ^0.0.143 load-json-file: ^7.0.1 @@ -16413,6 +16413,15 @@ __metadata: languageName: node linkType: hard +"js-tiktoken@npm:^1.0.8": + version: 1.0.8 + resolution: "js-tiktoken@npm:1.0.8" + dependencies: + base64-js: ^1.5.1 + checksum: ac6e666f14661b4e744bd60987e35275668863a686413bb35baca2d9a503bc95fcfa907f3c02b8e8abe4a4a70abe622c06cbb72ce3574b3b929d22f9146c3f85 + languageName: node + linkType: hard + "js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0": version: 4.0.0 resolution: "js-tokens@npm:4.0.0"