From 8d4292e6358920b2c9d8df49c6a154231c468512 Mon Sep 17 00:00:00 2001 From: Stainless Bot <107565488+stainless-bot@users.noreply.github.com> Date: Sat, 16 Dec 2023 19:47:08 -0500 Subject: [PATCH] feat(api): add token logprobs to chat completions (#576) --- api.md | 1 + examples/logprobs.ts | 23 ++++ src/index.ts | 1 + src/lib/ChatCompletionRunFunctions.test.ts | 15 +++ src/lib/ChatCompletionStream.ts | 30 ++++- src/resources/beta/threads/runs/steps.ts | 4 +- src/resources/chat/chat.ts | 1 + src/resources/chat/completions.ts | 110 ++++++++++++++++++- src/resources/chat/index.ts | 1 + src/resources/completions.ts | 11 +- src/resources/files.ts | 3 +- tests/api-resources/chat/completions.test.ts | 2 + 12 files changed, 184 insertions(+), 18 deletions(-) create mode 100755 examples/logprobs.ts diff --git a/api.md b/api.md index b8da4cf0c..82ffae114 100644 --- a/api.md +++ b/api.md @@ -37,6 +37,7 @@ Types: - ChatCompletionNamedToolChoice - ChatCompletionRole - ChatCompletionSystemMessageParam +- ChatCompletionTokenLogprob - ChatCompletionTool - ChatCompletionToolChoiceOption - ChatCompletionToolMessageParam diff --git a/examples/logprobs.ts b/examples/logprobs.ts new file mode 100755 index 000000000..5a4daf7de --- /dev/null +++ b/examples/logprobs.ts @@ -0,0 +1,23 @@ +#!/usr/bin/env -S npm run tsn -T + +import OpenAI from 'openai'; + +// gets API Key from environment variable OPENAI_API_KEY +const openai = new OpenAI(); + +async function main() { + const stream = await openai.beta.chat.completions + .stream({ + model: 'gpt-4', + messages: [{ role: 'user', content: 'Say this is a test' }], + stream: true, + logprobs: true, + }) + .on('logprob', (logprob) => { + console.log(logprob); + }); + + console.dir(await stream.finalChatCompletion(), { depth: null }); +} + +main(); diff --git a/src/index.ts b/src/index.ts index 71c1678b9..a03531dbe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -242,6 +242,7 @@ export namespace OpenAI { export import ChatCompletionNamedToolChoice = API.ChatCompletionNamedToolChoice; export import ChatCompletionRole = API.ChatCompletionRole; export import ChatCompletionSystemMessageParam = API.ChatCompletionSystemMessageParam; + export import ChatCompletionTokenLogprob = API.ChatCompletionTokenLogprob; export import ChatCompletionTool = API.ChatCompletionTool; export import ChatCompletionToolChoiceOption = API.ChatCompletionToolChoiceOption; export import ChatCompletionToolMessageParam = API.ChatCompletionToolMessageParam; diff --git a/src/lib/ChatCompletionRunFunctions.test.ts b/src/lib/ChatCompletionRunFunctions.test.ts index 2a5e91dcc..bb360b217 100644 --- a/src/lib/ChatCompletionRunFunctions.test.ts +++ b/src/lib/ChatCompletionRunFunctions.test.ts @@ -146,6 +146,7 @@ function* contentChoiceDeltas( yield { index, finish_reason: i === deltas.length - 1 ? 'stop' : null, + logprobs: null, delta: { role, content: deltas[i] ? `${deltas[i]}${i === deltas.length - 1 ? '' : ' '}` : null, @@ -593,6 +594,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -645,6 +647,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'stop', + logprobs: null, message: { role: 'assistant', content: `it's raining`, @@ -716,6 +719,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -808,6 +812,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -867,6 +872,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'stop', + logprobs: null, message: { role: 'assistant', content: `there are 3 properties in {"a": 1, "b": 2, "c": 3}`, @@ -953,6 +959,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -1006,6 +1013,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -1078,6 +1086,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'stop', + logprobs: null, message: { role: 'assistant', content: `there are 3 properties in {"a": 1, "b": 2, "c": 3}`, @@ -1164,6 +1173,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -1241,6 +1251,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -1291,6 +1302,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, message: { role: 'assistant', content: null, @@ -1360,6 +1372,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'stop', + logprobs: null, message: { role: 'assistant', content: `it's raining`, @@ -1436,6 +1449,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, delta: { role: 'assistant', content: null, @@ -2071,6 +2085,7 @@ describe('resource completions', () => { { index: 0, finish_reason: 'function_call', + logprobs: null, delta: { role: 'assistant', content: null, diff --git a/src/lib/ChatCompletionStream.ts b/src/lib/ChatCompletionStream.ts index b4534639f..c5b3efa6a 100644 --- a/src/lib/ChatCompletionStream.ts +++ b/src/lib/ChatCompletionStream.ts @@ -153,13 +153,22 @@ export class ChatCompletionStream Object.assign(snapshot, rest); } - for (const { delta, finish_reason, index, ...other } of chunk.choices) { + for (const { delta, finish_reason, index, logprobs = null, ...other } of chunk.choices) { let choice = snapshot.choices[index]; if (!choice) { - snapshot.choices[index] = { finish_reason, index, message: delta, ...other }; + snapshot.choices[index] = { finish_reason, index, message: delta, logprobs, ...other }; continue; } + if (logprobs) { + if (!choice.logprobs) { + choice.logprobs = logprobs; + } else if (logprobs.content) { + choice.logprobs.content ??= []; + choice.logprobs.content.push(...logprobs.content); + } + } + if (finish_reason) choice.finish_reason = finish_reason; Object.assign(choice, other); @@ -242,7 +251,7 @@ function finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletio const { id, choices, created, model } = snapshot; return { id, - choices: choices.map(({ message, finish_reason, index }): ChatCompletion.Choice => { + choices: choices.map(({ message, finish_reason, index, logprobs }): ChatCompletion.Choice => { if (!finish_reason) throw new OpenAIError(`missing finish_reason for choice ${index}`); const { content = null, function_call, tool_calls } = message; const role = message.role as 'assistant'; // this is what we expect; in theory it could be different which would make our types a slight lie but would be fine. @@ -251,12 +260,18 @@ function finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletio const { arguments: args, name } = function_call; if (args == null) throw new OpenAIError(`missing function_call.arguments for choice ${index}`); if (!name) throw new OpenAIError(`missing function_call.name for choice ${index}`); - return { message: { content, function_call: { arguments: args, name }, role }, finish_reason, index }; + return { + message: { content, function_call: { arguments: args, name }, role }, + finish_reason, + index, + logprobs, + }; } if (tool_calls) { return { index, finish_reason, + logprobs, message: { role, content, @@ -281,7 +296,7 @@ function finalizeChatCompletion(snapshot: ChatCompletionSnapshot): ChatCompletio }, }; } - return { message: { content: content, role }, finish_reason, index }; + return { message: { content: content, role }, finish_reason, index, logprobs }; }), created, model, @@ -336,6 +351,11 @@ export namespace ChatCompletionSnapshot { */ finish_reason: ChatCompletion.Choice['finish_reason'] | null; + /** + * Log probability information for the choice. + */ + logprobs: ChatCompletion.Choice.Logprobs | null; + /** * The index of the choice in the list of choices. */ diff --git a/src/resources/beta/threads/runs/steps.ts b/src/resources/beta/threads/runs/steps.ts index 9a335a1aa..618237c74 100644 --- a/src/resources/beta/threads/runs/steps.ts +++ b/src/resources/beta/threads/runs/steps.ts @@ -180,7 +180,7 @@ export interface MessageCreationStepDetails { message_creation: MessageCreationStepDetails.MessageCreation; /** - * Always `message_creation``. + * Always `message_creation`. */ type: 'message_creation'; } @@ -269,7 +269,7 @@ export interface RunStep { metadata: unknown | null; /** - * The object type, which is always `thread.run.step``. + * The object type, which is always `thread.run.step`. */ object: 'thread.run.step'; diff --git a/src/resources/chat/chat.ts b/src/resources/chat/chat.ts index 63e857e9b..07c7700dc 100644 --- a/src/resources/chat/chat.ts +++ b/src/resources/chat/chat.ts @@ -23,6 +23,7 @@ export namespace Chat { export import ChatCompletionNamedToolChoice = CompletionsAPI.ChatCompletionNamedToolChoice; export import ChatCompletionRole = CompletionsAPI.ChatCompletionRole; export import ChatCompletionSystemMessageParam = CompletionsAPI.ChatCompletionSystemMessageParam; + export import ChatCompletionTokenLogprob = CompletionsAPI.ChatCompletionTokenLogprob; export import ChatCompletionTool = CompletionsAPI.ChatCompletionTool; export import ChatCompletionToolChoiceOption = CompletionsAPI.ChatCompletionToolChoiceOption; export import ChatCompletionToolMessageParam = CompletionsAPI.ChatCompletionToolMessageParam; diff --git a/src/resources/chat/completions.ts b/src/resources/chat/completions.ts index 759c6e7c3..fce37ca53 100644 --- a/src/resources/chat/completions.ts +++ b/src/resources/chat/completions.ts @@ -96,11 +96,28 @@ export namespace ChatCompletion { */ index: number; + /** + * Log probability information for the choice. + */ + logprobs: Choice.Logprobs | null; + /** * A chat completion message generated by the model. */ message: ChatCompletionsAPI.ChatCompletionMessage; } + + export namespace Choice { + /** + * Log probability information for the choice. + */ + export interface Logprobs { + /** + * A list of message content tokens with log probability information. + */ + content: Array | null; + } + } } export interface ChatCompletionAssistantMessageParam { @@ -215,6 +232,11 @@ export namespace ChatCompletionChunk { * The index of the choice in the list of choices. */ index: number; + + /** + * Log probability information for the choice. + */ + logprobs?: Choice.Logprobs | null; } export namespace Choice { @@ -294,6 +316,16 @@ export namespace ChatCompletionChunk { } } } + + /** + * Log probability information for the choice. + */ + export interface Logprobs { + /** + * A list of message content tokens with log probability information. + */ + content: Array | null; + } } } @@ -350,7 +382,7 @@ export interface ChatCompletionFunctionMessageParam { /** * The contents of the function message. */ - content: string; + content: string | null; /** * The name of the function to call. @@ -499,6 +531,55 @@ export interface ChatCompletionSystemMessageParam { name?: string; } +export interface ChatCompletionTokenLogprob { + /** + * The token. + */ + token: string; + + /** + * A list of integers representing the UTF-8 bytes representation of the token. + * Useful in instances where characters are represented by multiple tokens and + * their byte representations must be combined to generate the correct text + * representation. Can be `null` if there is no bytes representation for the token. + */ + bytes: Array | null; + + /** + * The log probability of this token. + */ + logprob: number; + + /** + * List of the most likely tokens and their log probability, at this token + * position. In rare cases, there may be fewer than the number of requested + * `top_logprobs` returned. + */ + top_logprobs: Array; +} + +export namespace ChatCompletionTokenLogprob { + export interface TopLogprob { + /** + * The token. + */ + token: string; + + /** + * A list of integers representing the UTF-8 bytes representation of the token. + * Useful in instances where characters are represented by multiple tokens and + * their byte representations must be combined to generate the correct text + * representation. Can be `null` if there is no bytes representation for the token. + */ + bytes: Array | null; + + /** + * The log probability of this token. + */ + logprob: number; + } +} + export interface ChatCompletionTool { function: Shared.FunctionDefinition; @@ -612,7 +693,7 @@ export interface ChatCompletionCreateParamsBase { * particular function via `{"name": "my_function"}` forces the model to call that * function. * - * `none` is the default when no functions are present. `auto`` is the default if + * `none` is the default when no functions are present. `auto` is the default if * functions are present. */ function_call?: 'none' | 'auto' | ChatCompletionFunctionCallOption; @@ -637,7 +718,16 @@ export interface ChatCompletionCreateParamsBase { logit_bias?: Record | null; /** - * The maximum number of [tokens](/tokenizer) to generate in the chat completion. + * Whether to return log probabilities of the output tokens or not. If true, + * returns the log probabilities of each output token returned in the `content` of + * `message`. This option is currently not available on the `gpt-4-vision-preview` + * model. + */ + logprobs?: boolean | null; + + /** + * The maximum number of [tokens](/tokenizer) that can be generated in the chat + * completion. * * The total length of input tokens and generated tokens is limited by the model's * context length. @@ -663,7 +753,8 @@ export interface ChatCompletionCreateParamsBase { presence_penalty?: number | null; /** - * An object specifying the format that the model must output. + * An object specifying the format that the model must output. Compatible with + * `gpt-4-1106-preview` and `gpt-3.5-turbo-1106`. * * Setting to `{ "type": "json_object" }` enables JSON mode, which guarantees the * message the model generates is valid JSON. @@ -731,6 +822,13 @@ export interface ChatCompletionCreateParamsBase { */ tools?: Array; + /** + * An integer between 0 and 5 specifying the number of most likely tokens to return + * at each token position, each with an associated log probability. `logprobs` must + * be set to `true` if this parameter is used. + */ + top_logprobs?: number | null; + /** * An alternative to sampling with temperature, called nucleus sampling, where the * model considers the results of the tokens with top_p probability mass. So 0.1 @@ -775,7 +873,8 @@ export namespace ChatCompletionCreateParams { } /** - * An object specifying the format that the model must output. + * An object specifying the format that the model must output. Compatible with + * `gpt-4-1106-preview` and `gpt-3.5-turbo-1106`. * * Setting to `{ "type": "json_object" }` enables JSON mode, which guarantees the * message the model generates is valid JSON. @@ -854,6 +953,7 @@ export namespace Completions { export import ChatCompletionNamedToolChoice = ChatCompletionsAPI.ChatCompletionNamedToolChoice; export import ChatCompletionRole = ChatCompletionsAPI.ChatCompletionRole; export import ChatCompletionSystemMessageParam = ChatCompletionsAPI.ChatCompletionSystemMessageParam; + export import ChatCompletionTokenLogprob = ChatCompletionsAPI.ChatCompletionTokenLogprob; export import ChatCompletionTool = ChatCompletionsAPI.ChatCompletionTool; export import ChatCompletionToolChoiceOption = ChatCompletionsAPI.ChatCompletionToolChoiceOption; export import ChatCompletionToolMessageParam = ChatCompletionsAPI.ChatCompletionToolMessageParam; diff --git a/src/resources/chat/index.ts b/src/resources/chat/index.ts index ea9fe29bd..b8b69e453 100644 --- a/src/resources/chat/index.ts +++ b/src/resources/chat/index.ts @@ -16,6 +16,7 @@ export { ChatCompletionNamedToolChoice, ChatCompletionRole, ChatCompletionSystemMessageParam, + ChatCompletionTokenLogprob, ChatCompletionTool, ChatCompletionToolChoiceOption, ChatCompletionToolMessageParam, diff --git a/src/resources/completions.ts b/src/resources/completions.ts index f33624e73..00769fdbb 100644 --- a/src/resources/completions.ts +++ b/src/resources/completions.ts @@ -199,17 +199,18 @@ export interface CompletionCreateParamsBase { logit_bias?: Record | null; /** - * Include the log probabilities on the `logprobs` most likely tokens, as well the - * chosen tokens. For example, if `logprobs` is 5, the API will return a list of - * the 5 most likely tokens. The API will always return the `logprob` of the - * sampled token, so there may be up to `logprobs+1` elements in the response. + * Include the log probabilities on the `logprobs` most likely output tokens, as + * well the chosen tokens. For example, if `logprobs` is 5, the API will return a + * list of the 5 most likely tokens. The API will always return the `logprob` of + * the sampled token, so there may be up to `logprobs+1` elements in the response. * * The maximum value for `logprobs` is 5. */ logprobs?: number | null; /** - * The maximum number of [tokens](/tokenizer) to generate in the completion. + * The maximum number of [tokens](/tokenizer) that can be generated in the + * completion. * * The token count of your prompt plus `max_tokens` cannot exceed the model's * context length. diff --git a/src/resources/files.ts b/src/resources/files.ts index ea3f3b9c1..db8f3a66a 100644 --- a/src/resources/files.ts +++ b/src/resources/files.ts @@ -15,7 +15,8 @@ export class Files extends APIResource { * Upload a file that can be used across various endpoints. The size of all the * files uploaded by one organization can be up to 100 GB. * - * The size of individual files can be a maximum of 512 MB. See the + * The size of individual files can be a maximum of 512 MB or 2 million tokens for + * Assistants. See the * [Assistants Tools guide](https://platform.openai.com/docs/assistants/tools) to * learn more about the types of files supported. The Fine-tuning API only supports * `.jsonl` files. diff --git a/tests/api-resources/chat/completions.test.ts b/tests/api-resources/chat/completions.test.ts index 15b815a51..49f3562b0 100644 --- a/tests/api-resources/chat/completions.test.ts +++ b/tests/api-resources/chat/completions.test.ts @@ -31,6 +31,7 @@ describe('resource completions', () => { function_call: 'none', functions: [{ description: 'string', name: 'string', parameters: { foo: 'bar' } }], logit_bias: { foo: 0 }, + logprobs: true, max_tokens: 0, n: 1, presence_penalty: -2, @@ -45,6 +46,7 @@ describe('resource completions', () => { { type: 'function', function: { description: 'string', name: 'string', parameters: { foo: 'bar' } } }, { type: 'function', function: { description: 'string', name: 'string', parameters: { foo: 'bar' } } }, ], + top_logprobs: 0, top_p: 1, user: 'user-1234', });