From 969d09f91fa16e0e25bd622ee741638edb09df27 Mon Sep 17 00:00:00 2001 From: Peter Salas Date: Tue, 9 Apr 2024 11:50:23 -0700 Subject: [PATCH] Update AnthropicChatModel, relax OpenAIChatModel --- packages/ai-jsx/package.json | 4 +- .../src/batteries/constrained-output.tsx | 2 +- packages/ai-jsx/src/core/completion.tsx | 12 + packages/ai-jsx/src/lib/anthropic.tsx | 354 ++++++++++++---- packages/ai-jsx/src/lib/openai.tsx | 57 +-- packages/docs/docs/changelog.md | 5 + packages/examples/package.json | 2 +- packages/examples/src/chat-function-call.tsx | 5 +- packages/examples/test/core/completion.tsx | 382 ++++++++++++++---- packages/examples/test/lib/openai.tsx | 6 +- yarn.lock | 14 +- 11 files changed, 627 insertions(+), 216 deletions(-) diff --git a/packages/ai-jsx/package.json b/packages/ai-jsx/package.json index 86d7aec73..d6a5066a4 100644 --- a/packages/ai-jsx/package.json +++ b/packages/ai-jsx/package.json @@ -4,7 +4,7 @@ "repository": "fixie-ai/ai-jsx", "bugs": "https://github.com/fixie-ai/ai-jsx/issues", "homepage": "https://ai-jsx.com", - "version": "0.29.0", + "version": "0.30.0", "volta": { "extends": "../../package.json" }, @@ -375,7 +375,7 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" }, "dependencies": { - "@anthropic-ai/sdk": "^0.5.10", + "@anthropic-ai/sdk": "^0.20.1", "@mdx-js/mdx": "^2.3.0", "@nick.heiner/wandb-fork": "^0.5.2-5", "@opentelemetry/api": "^1.4.1", diff --git a/packages/ai-jsx/src/batteries/constrained-output.tsx b/packages/ai-jsx/src/batteries/constrained-output.tsx index 22a4103d3..59a67f901 100644 --- a/packages/ai-jsx/src/batteries/constrained-output.tsx +++ b/packages/ai-jsx/src/batteries/constrained-output.tsx @@ -323,7 +323,7 @@ export async function* JsonChatCompletionFunctionCall( functionDefinitions={{ print: { description: 'Prints the response in a human readable format.', - parameters: schema, + parameters: schema as any, }, }} forcedFunction="print" diff --git a/packages/ai-jsx/src/core/completion.tsx b/packages/ai-jsx/src/core/completion.tsx index b2a46b97f..445d922fb 100644 --- a/packages/ai-jsx/src/core/completion.tsx +++ b/packages/ai-jsx/src/core/completion.tsx @@ -29,6 +29,8 @@ export interface ModelProps { maxTokens?: number; /** The number of tokens to reserve for the generation. */ reservedTokens?: number; + /** Maximum number of input tokens to allow. */ + maxInputTokens?: number; /** A list of stop tokens. */ stop?: string[]; @@ -37,6 +39,16 @@ export interface ModelProps { * * @see https://platform.openai.com/docs/api-reference/chat/create#chat/create-top_p */ topP?: number; + + /** + * Any function definitions (tools) that the model can choose to invoke. + */ + functionDefinitions?: Record; + + /** + * If specified, the model will be forced to use this function. + */ + forcedFunction?: string; } /** diff --git a/packages/ai-jsx/src/lib/anthropic.tsx b/packages/ai-jsx/src/lib/anthropic.tsx index c95c799fc..80c019ed2 100644 --- a/packages/ai-jsx/src/lib/anthropic.tsx +++ b/packages/ai-jsx/src/lib/anthropic.tsx @@ -2,8 +2,8 @@ import AnthropicSDK from '@anthropic-ai/sdk'; import { getEnvVar } from './util.js'; import * as AI from '../index.js'; import { Node } from '../index.js'; -import { ChatProvider, ModelProps, ModelPropsWithChildren } from '../core/completion.js'; -import { AssistantMessage, ConversationMessage, UserMessage, renderToConversation } from '../core/conversation.js'; +import { ChatProvider, ModelPropsWithChildren } from '../core/completion.js'; +import { AssistantMessage, FunctionCall, renderToConversation } from '../core/conversation.js'; import { AIJSXError, ErrorCode } from '../core/errors.js'; import { debugRepresentation } from '../core/debug.js'; import _ from 'lodash'; @@ -24,21 +24,7 @@ const anthropicClientContext = AI.createContext<() => AnthropicSDK>( * * @see https://docs.anthropic.com/claude/reference/complete_post. */ -export type ValidChatModel = - | 'claude-1' - | 'claude-1-100k' - | 'claude-instant-1' - | 'claude-instant-1-100k' - | 'claude-1.3' - | 'claude-1.3-100k' - | 'claude-1.2' - | 'claude-1.0' - | 'claude-instant-1.2' - | 'claude-instant-1.1' - | 'claude-instant-1.1-100k' - | 'claude-instant-1.0' - | 'claude-2' - | 'claude-2.0'; +export type ValidChatModel = AnthropicSDK.MessageCreateParamsStreaming['model']; /** * If you use an Anthropic model without specifying the max tokens for the completion, this value will be used as the default. @@ -58,7 +44,10 @@ export function Anthropic({ client, completionModel, ...defaults -}: { children: Node; client?: AnthropicSDK; chatModel?: ValidChatModel; completionModel?: never } & ModelProps) { +}: { children: Node; client?: AnthropicSDK; chatModel?: ValidChatModel; completionModel?: never } & Omit< + AnthropicChatModelProps, + 'children' | 'model' +>) { let result = children; if (client) { @@ -88,79 +77,270 @@ export function Anthropic({ interface AnthropicChatModelProps extends ModelPropsWithChildren { model: ValidChatModel; + useBetaTools?: boolean; } export async function* AnthropicChatModel( props: AnthropicChatModelProps, { render, getContext, logger, memo }: AI.ComponentContext ): AI.RenderableStream { - if ('functionDefinitions' in props && props.functionDefinitions) { + yield AI.AppendOnlyStream; + + const anthropic = getContext(anthropicClientContext)(); + const messages = await renderToConversation(props.children, render, logger, 'prompt'); + + const legacyModels: Record = { + 'claude-1': 'claude-1.3', + 'claude-1-100k': 'claude-1.3', + 'claude-2': 'claude-2.1', + 'claude-instant-1': 'claude-instant-1.2', + 'claude-instant-1-100k': 'claude-instant-1.2', + 'claude-instant-1.1-100k': 'claude-instant-1.1', + }; + const resolvedModel = props.model in legacyModels ? legacyModels[props.model] : props.model; + /** + * From https://docs.anthropic.com/claude/docs/legacy-model-guide#anthropics-legacy-models + * > Our legacy models include Claude Instant 1.2, Claude 2.0, and Claude 2.1. Of these legacy models, Claude 2.1 is the only model with system prompt support (all Claude 3 models have full system prompt support). + */ + const supportsSystemPrompt = !( + resolvedModel.startsWith('claude-1') || + resolvedModel.startsWith('claude-instant') || + resolvedModel.startsWith('claude-2.0') + ); + const leadingSystemMessages = _.takeWhile(messages, (m) => supportsSystemPrompt && m.type === 'system'); + const commonRequestProperties = { + system: (await Promise.all(leadingSystemMessages.map((m) => render(m.element)))).join('\n\n') || undefined, + model: resolvedModel, + max_tokens: props.maxTokens ?? defaultMaxTokens, + temperature: props.temperature, + stop_sequences: props.stop, + top_p: props.topP, + }; + + type MessageParamWithoutStrings = AnthropicSDK.Beta.Tools.ToolsBetaMessageParam & { content: unknown[] }; + + const hasFunctionDefinitions = Object.keys(props.functionDefinitions ?? {}).length > 0; + const useBetaTools = props.useBetaTools ?? hasFunctionDefinitions; + + const anthropicMessages: MessageParamWithoutStrings[] = ( + await Promise.all( + messages + .slice(leadingSystemMessages.length) + .map>(async (m) => { + switch (m.type) { + case 'user': + case 'assistant': + return { role: m.type, content: [{ type: 'text', text: await render(m.element) }] }; + case 'system': { + // Polyfill system messages that are either not supported or do not appear at the start of the prompt. + const text = await render(m.element); + return [ + { + role: 'user', + content: [ + { + type: 'text', + text: `For subsequent replies you will adhere to the following instructions: ${text}`, + }, + ], + }, + { role: 'assistant', content: [{ type: 'text', text: 'Okay, I will do that.' }] }, + ]; + } + case 'functionCall': + return { + role: 'assistant', + content: useBetaTools + ? [ + { + type: 'tool_use', + id: m.element.props.id!, + name: m.element.props.name, + input: m.element.props.args, + }, + ] + : [ + { + type: 'text', + text: await render( + <> + {'<'}function_calls{'>'} + {'\n'} + {'<'}invoke{'>'} + {'\n'} + {'<'}tool_name{'>'} + {m.element.props.name} + {'<'}/tool_name{'>'} + {'\n'} + {'<'}parameters{'>'} + {'\n'} + {Object.entries(m.element.props.args).map(([key, value]) => ( + <> + {'<'} + {key} + {'>'} + {value} + {'<'}/{key} + {'>'} + {'\n'} + + ))} + {'<'}/parameters{'>'} + {'\n'} + {'<'}/invoke{'>'} + {'\n'} + {'<'}/function_calls{'>'} + + ), + }, + ], + }; + case 'functionResponse': + return { + role: 'user', + content: useBetaTools + ? [ + { + type: 'tool_result', + tool_use_id: m.element.props.id!, + content: [{ type: 'text', text: await render(m.element.props.children) }], + is_error: m.element.props.failed, + }, + ] + : [ + { + type: 'text', + text: await render( + <> + {'<'}function_results{'>'} + {'\n'} + {'<'}result{'>'} + {'\n'} + {'<'}tool_name{'>'} + {m.element.props.name} + {'<'}/tool_name{'>'} + {'\n'} + {m.element.props.failed ? ( + <> + {'<'}error{'>'} + {'\n'} + {m.element.props.children} + {''} + {'\n'} + + ) : ( + <> + {'<'}stdout{'>'} + {'\n'} + {m.element.props.children} + {'\n'} + {''} + {'\n'} + + )} + {'<'}/invoke{'>'} + {'\n'} + {'<'}/function_results{'>'} + + ), + }, + ], + }; + default: + return [] as MessageParamWithoutStrings[]; + } + }) + ) + ).flat(1); + + if (props.forcedFunction && props.functionDefinitions && props.forcedFunction in props.functionDefinitions) { + const polyfillPrompt: MessageParamWithoutStrings = { + role: 'user', + content: [ + { + type: 'text', + text: `Use the \`${props.forcedFunction}\` tool.`, + }, + ], + }; + logger.warn( + { forcedFunction: props.forcedFunction, addedUserMessage: polyfillPrompt }, + 'Anthropic does not directly support forced functions. Adding additional user message to prompt.' + ); + anthropicMessages.push(polyfillPrompt); + } else if (props.forcedFunction) { throw new AIJSXError( - 'Anthropic does not support function calling, but function definitions were provided.', - ErrorCode.ChatModelDoesNotSupportFunctions, + `The function ${props.forcedFunction} was forced, but no function with that name was defined.`, + ErrorCode.ChatCompletionBadInput, 'user' ); } - yield AI.AppendOnlyStream; - const messages = await Promise.all( - // TODO: Support token budget/conversation shrinking - ( - await renderToConversation(props.children, render, logger, 'prompt') - ) - .flatMap>((message) => { - if (message.type === 'system') { - return [ - { - type: 'user', - element: ( - - For subsequent replies you will adhere to the following instructions: {message.element} - - ), - }, - { type: 'assistant', element: Okay, I will do that. }, - ]; - } - - return [message]; - }) - .map(async (message) => { - switch (message.type) { - case 'user': - return `${AnthropicSDK.HUMAN_PROMPT} ${await render(message.element)}`; - case 'assistant': - case 'functionCall': - case 'functionResponse': - return `${AnthropicSDK.AI_PROMPT} ${await render(message.element)}`; - } - }) - ); - if (!messages.length) { + // Combine any adjacent user and assistant messages. + const combinedAnthropicMessages: typeof anthropicMessages = []; + for (const message of anthropicMessages) { + const lastMessage = combinedAnthropicMessages.at(-1); + if (lastMessage?.role === message.role) { + lastMessage.content.push(...message.content); + } else { + combinedAnthropicMessages.push(message); + } + } + + if (useBetaTools) { + // If there are tools in the prompt, we need to use the tools API (which currently does not support streaming). + const toolsRequest: AnthropicSDK.Beta.Tools.MessageCreateParamsNonStreaming = { + messages: combinedAnthropicMessages, + tools: Object.entries(props.functionDefinitions ?? {}).map(([functionName, definition]) => ({ + name: functionName, + description: definition.description, + input_schema: definition.parameters as any, + })), + stream: false, + ...commonRequestProperties, + }; + logger.debug({ toolsRequest }, 'Calling anthropic.beta.tools.messages.create'); + try { + const result = await anthropic.beta.tools.messages.create(toolsRequest); + return result.content.map((contentBlock) => + contentBlock.type === 'text' ? ( + {contentBlock.text} + ) : ( + + ) + ); + } catch (err) { + if (err instanceof AnthropicSDK.APIError) { + throw new AIJSXError( + err.message, + ErrorCode.AnthropicAPIError, + 'runtime', + Object.fromEntries(Object.entries(err)) + ); + } + throw err; + } + } + + if (hasFunctionDefinitions) { throw new AIJSXError( - "ChatCompletion must have at least one child that's UserMessage or AssistantMessage, but no such children were found.", - ErrorCode.ChatCompletionMissingChildren, + 'Anthropic models only support functions via the beta tools API, but useBetaTools was set to false.', + ErrorCode.ChatModelDoesNotSupportFunctions, 'user' ); } - messages.push(AnthropicSDK.AI_PROMPT); - - const anthropic = getContext(anthropicClientContext)(); - const anthropicCompletionRequest: AnthropicSDK.CompletionCreateParams = { - prompt: messages.join('\n\n'), - max_tokens_to_sample: props.maxTokens ?? defaultMaxTokens, - temperature: props.temperature, - model: props.model, - stop_sequences: props.stop, + const anthropicCompletionRequest: AnthropicSDK.MessageCreateParamsStreaming = { + messages: combinedAnthropicMessages as AnthropicSDK.MessageParam[], + ...commonRequestProperties, stream: true, - top_p: props.topP, }; - logger.debug({ anthropicCompletionRequest }, 'Calling createCompletion'); + logger.debug({ anthropicCompletionRequest }, 'Calling anthropic.messages.create'); - let response: Awaited>; + const responsePromise = anthropic.messages.create(anthropicCompletionRequest); + let response: Awaited; try { - response = await anthropic.completions.create(anthropicCompletionRequest); + response = await anthropic.messages.create(anthropicCompletionRequest); } catch (err) { if (err instanceof AnthropicSDK.APIError) { throw new AIJSXError( @@ -178,18 +358,28 @@ export async function* AnthropicChatModel( let complete = false; const Stream = async function* (): AI.RenderableStream { yield AI.AppendOnlyStream; - let isFirstResponse = true; - for await (const completion of response) { - let text = completion.completion; - if (isFirstResponse && text.length > 0) { - isFirstResponse = false; - if (text.startsWith(' ')) { - text = text.slice(1); - } + for await (const deltaEvent of response) { + logger.trace({ deltaEvent }, 'Got Anthropic stream event'); + switch (deltaEvent.type) { + case 'message_start': + case 'message_stop': + case 'content_block_start': + case 'content_block_stop': + break; + case 'message_delta': + logger.setAttribute('anthropic.usage', JSON.stringify(deltaEvent.usage)); + if (deltaEvent.delta.stop_reason) { + logger.setAttribute('anthropic.stop_reason', deltaEvent.delta.stop_reason); + } + break; + case 'content_block_delta': + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (deltaEvent.delta.type === 'text_delta') { + accumulatedContent += deltaEvent.delta.text; + yield deltaEvent.delta.text; + } + break; } - accumulatedContent += text; - logger.trace({ completion }, 'Got Anthropic stream event'); - yield text; } complete = true; diff --git a/packages/ai-jsx/src/lib/openai.tsx b/packages/ai-jsx/src/lib/openai.tsx index d06355709..17dcf6ce6 100644 --- a/packages/ai-jsx/src/lib/openai.tsx +++ b/packages/ai-jsx/src/lib/openai.tsx @@ -3,7 +3,7 @@ * @packageDocumentation */ -import { Jsonifiable, MergeExclusive } from 'type-fest'; +import { Jsonifiable } from 'type-fest'; import { ChatProvider, CompletionProvider, @@ -41,6 +41,8 @@ export type ValidChatModel = | 'gpt-4-32k-0314' // discontinue on 06/13/2024 | 'gpt-4-32k-0613' | 'gpt-4-1106-preview' + | 'gpt-4-0125-preview' + | 'gpt-4-turbo-preview' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-0301' // discontinue on 06/13/2024 | 'gpt-3.5-turbo-0613' @@ -156,15 +158,6 @@ function logitBiasOfTokens(tokens: Record) { ); } -/** - * Returns true if the given model supports function calling. - * @param model The model to check. - * @returns True if the model supports function calling, false otherwise. - */ -function chatModelSupportsFunctions(model: ValidChatModel) { - return model.startsWith('gpt-4') || model.startsWith('gpt-3.5-turbo'); -} - /** * Represents an OpenAI text completion model (e.g., `text-davinci-003`). */ @@ -225,10 +218,9 @@ function estimateFunctionTokenCount(functions: Record -): number | undefined { +): number { const TOKENS_CONSUMED_BY_REPLY_PREFIX = 3; - const functionEstimate = - chatModelSupportsFunctions(model) && functionDefinitions ? estimateFunctionTokenCount(functionDefinitions) : 0; + const functionEstimate = functionDefinitions ? estimateFunctionTokenCount(functionDefinitions) : 0; switch (model) { case 'gpt-4': @@ -240,11 +232,13 @@ function tokenLimitForChatModel( case 'gpt-4-32k-0613': return 32768 - functionEstimate - TOKENS_CONSUMED_BY_REPLY_PREFIX; case 'gpt-4-1106-preview': + case 'gpt-4-0125-preview': + case 'gpt-4-turbo-preview': return 128_000 - functionEstimate - TOKENS_CONSUMED_BY_REPLY_PREFIX; - case 'gpt-3.5-turbo': case 'gpt-3.5-turbo-0301': case 'gpt-3.5-turbo-0613': return 4096 - functionEstimate - TOKENS_CONSUMED_BY_REPLY_PREFIX; + case 'gpt-3.5-turbo': case 'gpt-3.5-turbo-16k': case 'gpt-3.5-turbo-16k-0613': case 'gpt-3.5-turbo-1106': @@ -252,12 +246,12 @@ function tokenLimitForChatModel( default: { // eslint-disable-next-line @typescript-eslint/no-unused-vars const _: never = model; - return undefined; + return 16384 - functionEstimate - TOKENS_CONSUMED_BY_REPLY_PREFIX; } } } -async function tokenCountForConversationMessage( +export async function tokenCountForConversationMessage( message: ConversationMessage, render: AI.RenderContext['render'] ): Promise { @@ -377,29 +371,10 @@ export async function* OpenAIChatModel( props: ModelPropsWithChildren & { model: ValidChatModel; logitBias?: Record; - } & MergeExclusive< - { - functionDefinitions: Record; - forcedFunction: string; - }, - { - functionDefinitions?: never; - forcedFunction?: never; - } - >, + }, { render, getContext, logger, memo }: AI.ComponentContext ): AI.RenderableStream { - if (props.functionDefinitions) { - if (!chatModelSupportsFunctions(props.model)) { - throw new AIJSXError( - `The ${props.model} model does not support function calling, but function definitions were provided.`, - ErrorCode.ChatModelDoesNotSupportFunctions, - 'user' - ); - } - } - - if (props.forcedFunction && !Object.keys(props.functionDefinitions).includes(props.forcedFunction)) { + if (props.forcedFunction && !Object.keys(props.functionDefinitions ?? {}).includes(props.forcedFunction)) { throw new AIJSXError( `The function ${props.forcedFunction} was forced, but no function with that name was defined.`, ErrorCode.ChatCompletionBadInput, @@ -409,12 +384,8 @@ export async function* OpenAIChatModel( yield AI.AppendOnlyStream; - let promptTokenLimit = tokenLimitForChatModel(props.model, props.functionDefinitions); - - // If reservedTokens (or maxTokens) is set, reserve that many tokens for the reply. - if (promptTokenLimit !== undefined) { - promptTokenLimit -= props.reservedTokens ?? props.maxTokens ?? 0; - } + const modelTokenLimit = tokenLimitForChatModel(props.model, props.functionDefinitions); + const promptTokenLimit = props.maxInputTokens ?? modelTokenLimit - (props.reservedTokens ?? props.maxTokens ?? 0); const conversationMessages = await renderToConversation( props.children, diff --git a/packages/docs/docs/changelog.md b/packages/docs/docs/changelog.md index 66a4050c0..6ed2a5702 100644 --- a/packages/docs/docs/changelog.md +++ b/packages/docs/docs/changelog.md @@ -2,6 +2,11 @@ ## 0.30.0 +- Added support for Claude 3 and messages API in `AnthropicChatModel` +- Relaxed function calling check for `OpenAIChatModel` + +## [0.29.0](https://github.com/fixie-ai/ai-jsx/tree/4ce9b17471ff6cd8e3928d189d254dca8e65e9ba) + - Changed the `FunctionDefinition` and `Tool` types to explicit JSON Schema. Zod types must now be explicitly converted to JSON Schema, and the `required` semantics now match JSON Schema. diff --git a/packages/examples/package.json b/packages/examples/package.json index 1760e0fe6..285af4a5c 100644 --- a/packages/examples/package.json +++ b/packages/examples/package.json @@ -72,7 +72,7 @@ "test": "yarn run typecheck && yarn run unit" }, "dependencies": { - "@anthropic-ai/sdk": "^0.5.10", + "@anthropic-ai/sdk": "^0.20.1", "@opentelemetry/api": "^1.4.1", "@opentelemetry/api-logs": "^0.41.1", "@opentelemetry/exporter-logs-otlp-grpc": "^0.41.1", diff --git a/packages/examples/src/chat-function-call.tsx b/packages/examples/src/chat-function-call.tsx index 51d0839f9..2e0bc88b1 100644 --- a/packages/examples/src/chat-function-call.tsx +++ b/packages/examples/src/chat-function-call.tsx @@ -2,6 +2,7 @@ import { showInspector } from 'ai-jsx/core/inspector'; import { ChatCompletion, ChatProvider, + FunctionDefinition, SystemMessage, UserMessage, FunctionCall, @@ -22,7 +23,7 @@ function ModelProducesFunctionCall({ query }: { query: string }) { required: true, }, }, - }, + } as FunctionDefinition, }} > You are a tool that may use functions to answer a user question. @@ -46,7 +47,7 @@ function ModelProducesFinalResponse({ query }: { query: string }) { required: true, }, }, - }, + } as FunctionDefinition, }} > You are a tool that may use functions to answer a user question. diff --git a/packages/examples/test/core/completion.tsx b/packages/examples/test/core/completion.tsx index d68d170a7..9d378f5e9 100644 --- a/packages/examples/test/core/completion.tsx +++ b/packages/examples/test/core/completion.tsx @@ -31,15 +31,19 @@ jestFetchMock.enableFetchMocks(); process.env.OPENAI_API_KEY = 'fake-openai-key'; process.env.ANTHROPIC_API_KEY = 'fake-anthropic-key'; -import nock from 'nock'; - import * as AI from 'ai-jsx'; import { ChatCompletion } from 'ai-jsx/core/completion'; -import { FunctionCall, FunctionResponse, UserMessage, SystemMessage, Shrinkable } from 'ai-jsx/core/conversation'; +import { + FunctionCall, + FunctionResponse, + UserMessage, + SystemMessage, + Shrinkable, + AssistantMessage, +} from 'ai-jsx/core/conversation'; import { OpenAI, OpenAIClient } from 'ai-jsx/lib/openai'; import { Tool } from 'ai-jsx/batteries/use-tools'; import { Anthropic } from 'ai-jsx/lib/anthropic'; -import { CompletionCreateParams } from '@anthropic-ai/sdk/resources/completions'; import { Jsonifiable } from 'type-fest'; import { NodeSDK } from '@opentelemetry/sdk-node'; @@ -82,7 +86,7 @@ describe('OpenTelemetry', () => { {"hello"} ]", "ai.jsx.tag": "ShrinkConversation", - "ai.jsx.tree": " + "ai.jsx.tree": " {"hello"} @@ -167,7 +171,7 @@ describe('OpenTelemetry', () => { {"hello"} ]", "ai.jsx.tag": "ShrinkConversation", - "ai.jsx.tree": " + "ai.jsx.tree": " {"hello"} @@ -353,7 +357,7 @@ it('throws an error when a bare string is passsed as a replacement', async () => await expect(() => AI.createRenderContext().render( - + {largeString} @@ -450,52 +454,313 @@ describe('functions', () => { expect(result).toEqual('response from OpenAI'); }); - - it('throws an error for models that do not support functions', () => - expect(() => - AI.createRenderContext().render( - - - Hello - - - ) - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Anthropic does not support function calling, but function definitions were provided."` - )); }); describe('anthropic', () => { - it('handles function calls/responses', async () => { - const handleRequest = jest.fn(); - mockAnthropicResponse('response from Anthropic', handleRequest); + it('adapts message types correctly', async () => { const result = await AI.createRenderContext().render( - + + +myFunc + +option1 + + +", + "type": "text", + }, + ], + "role": "assistant", + }, + { + "content": [ + { + "text": " + +myFunc + +12345 + + +", + "type": "text", + }, + ], + "role": "user", + }, + ], + "model": "claude-3-haiku-20240307", + "stop_sequences": undefined, + "stream": true, + "system": "This is a system message. + +And this is another one.", + "temperature": undefined, + "top_p": undefined, +} +`); + yield { type: 'content_block_delta', delta: { type: 'text_delta', text: 'response from Anthropic' } }; + }, + }, + }} + > + This is a system message. + And this is another one. Hello - - 12345 + Hi there! + How are you? + This is a system message in the middle of the conversation. + Good. + And how are you? + + + 12345 + ); - expect(handleRequest.mock.calls[0][0].prompt).toMatchInlineSnapshot(` - " - - Human: Hello - - - - Assistant: Call function myFunc with {"myParam":"option1"} - - - - Assistant: function myFunc returned 12345 - - + expect(result).toEqual('response from Anthropic'); + }); - Assistant:" + it('adapts message types correctly to the beta tools api', async () => { + const result = await AI.createRenderContext().render( + + + This is a system message. + And this is another one. + Hello + Hi there! + How are you? + This is a system message in the middle of the conversation. + Good. + And how are you? + + + 12345 + + + + ); + expect(result).toMatchInlineSnapshot(` + "response from Anthropic + Call function myFunc (id tool1) with {"myParam":"option1"}" `); - expect(result).toEqual('response from Anthropic'); }); }); @@ -548,36 +813,3 @@ function mockOpenAIResponse(message: string, handleRequest?: jest.MockedFn<(req: } ); } - -function mockAnthropicResponse( - message: string, - handleRequest?: jest.MockedFn<(req: CompletionCreateParams) => Promise> -) { - const SSE_PREFIX = 'data: '; - const SSE_TERMINATOR = '\n\n'; - - nock('https://api.anthropic.com') - .post('/v1/complete') - .reply(200, (_uri: string, requestBody: CompletionCreateParams) => { - handleRequest?.(requestBody); - - function createDelta(messagePart: string) { - const response = { - completion: messagePart, - stop_reason: null, - model: requestBody.model, - }; - return `event: completion\n${SSE_PREFIX}${JSON.stringify(response)}${SSE_TERMINATOR}`; - } - - const completionEvents = message.split('').map(createDelta).join('\n'); - - const finalResponse = { - completion: '', - stop_reason: 'stop_sequence', - model: requestBody.model, - }; - - return `${completionEvents}event: completion\n${SSE_PREFIX}${JSON.stringify(finalResponse)}${SSE_TERMINATOR}`; - }); -} diff --git a/packages/examples/test/lib/openai.tsx b/packages/examples/test/lib/openai.tsx index 0da693e7d..eee3f0015 100644 --- a/packages/examples/test/lib/openai.tsx +++ b/packages/examples/test/lib/openai.tsx @@ -14,7 +14,7 @@ describe('OpenAIChatModel', () => { chat: { completions: { async *create(req) { - expect(req.max_tokens).toBe(4096); + expect(req.max_tokens).toBe(10 ** 10); expect(req.messages).toEqual([ expect.objectContaining({ content: 'Hello!', @@ -27,7 +27,7 @@ describe('OpenAIChatModel', () => { }, }} > - + Hello!}> This should be replaced @@ -61,7 +61,7 @@ describe('OpenAIChatModel', () => { }, }} > - + Hello!}> This should be replaced diff --git a/yarn.lock b/yarn.lock index 4d350a33a..cf8f5d677 100644 --- a/yarn.lock +++ b/yarn.lock @@ -211,19 +211,19 @@ __metadata: languageName: node linkType: hard -"@anthropic-ai/sdk@npm:^0.5.10": - version: 0.5.10 - resolution: "@anthropic-ai/sdk@npm:0.5.10" +"@anthropic-ai/sdk@npm:^0.20.1": + version: 0.20.1 + resolution: "@anthropic-ai/sdk@npm:0.20.1" dependencies: "@types/node": ^18.11.18 "@types/node-fetch": ^2.6.4 abort-controller: ^3.0.0 agentkeepalive: ^4.2.1 - digest-fetch: ^1.3.0 form-data-encoder: 1.7.2 formdata-node: ^4.3.2 node-fetch: ^2.6.7 - checksum: 4f70b06be7be7e9baee0d2b85648f17643a58c89003331a9b38dea8e3eaea3519d60ac5ef2074b903897214f53358b92c6e87e7ea4f22787410cccb85d5a8b1b + web-streams-polyfill: ^3.2.1 + checksum: a880088ffeb993ea835f3ec250d53bf6ba23e97c3dfc54c915843aa8cb4778849fb7b85de0a359155c36595a5a5cc1db64139d407d2e36a2423284ebfe763cce languageName: node linkType: hard @@ -7107,7 +7107,7 @@ __metadata: version: 0.0.0-use.local resolution: "ai-jsx@workspace:packages/ai-jsx" dependencies: - "@anthropic-ai/sdk": ^0.5.10 + "@anthropic-ai/sdk": ^0.20.1 "@jest/globals": ^29.5.0 "@mdx-js/mdx": ^2.3.0 "@nick.heiner/wandb-fork": ^0.5.2-5 @@ -12111,7 +12111,7 @@ __metadata: version: 0.0.0-use.local resolution: "examples@workspace:packages/examples" dependencies: - "@anthropic-ai/sdk": ^0.5.10 + "@anthropic-ai/sdk": ^0.20.1 "@jest/globals": ^29.5.0 "@opentelemetry/api": ^1.4.1 "@opentelemetry/api-logs": ^0.41.1