feat: added o1-preview and o1-mini chat completion models (#56)

src/GptEncoding.ts

@@ -206,11 +206,6 @@ export class GptEncoding {
   }
 
   encode(lineToEncode: string, encodeOptions?: EncodeOptions): number[] {
-    // const encodedTokens: number[] = []
-    // for (const tokens of this.encodeGenerator(lineToEncode, encodeOptions)) {
-    //   encodedTokens.push(...tokens)
-    // }
-    // return encodedTokens
     const specialTokenConfig = encodeOptions
       ? this.processSpecialTokens(encodeOptions)
       : this.defaultSpecialTokenConfig

src/mapping.ts

@@ -17,6 +17,8 @@ export const encodingNames = [
 ] as const
 
 const chatEnabledModelsMap = {
+  'o1-preview': o200k_base,
+  'o1-mini': o200k_base,
   'gpt-4': cl100k_base,
   'gpt-4-0314': cl100k_base,
   'gpt-4-0613': cl100k_base,

src/model/o1-mini.ts

@@ -0,0 +1,29 @@
+/* eslint-disable import/extensions */
+import encoder from '../encodings/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('o1-mini', () => encoder)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api

src/model/o1-preview.ts

@@ -0,0 +1,29 @@
+/* eslint-disable import/extensions */
+import encoder from '../encodings/o200k_base.js'
+import { GptEncoding } from '../GptEncoding.js'
+
+export * from '../specialTokens.js'
+// prettier-ignore
+const api = GptEncoding.getEncodingApiForModel('o1-preview', () => encoder)
+const {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeGenerator,
+  isWithinTokenLimit,
+  encodeChat,
+  encodeChatGenerator,
+} = api
+export {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  encodeChat,
+  encodeChatGenerator,
+  encodeGenerator,
+  isWithinTokenLimit,
+}
+// eslint-disable-next-line import/no-default-export
+export default api