From 52c8549d6eb82da782724801866e5ded9db4acbf Mon Sep 17 00:00:00 2001 From: Peter Salas Date: Thu, 16 Nov 2023 14:10:38 -0800 Subject: [PATCH 1/6] Introduce transcript newlines in the place of any function calls/responses --- packages/voice/src/app/agent/chat.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/voice/src/app/agent/chat.tsx b/packages/voice/src/app/agent/chat.tsx index 41489ba0..f455f629 100644 --- a/packages/voice/src/app/agent/chat.tsx +++ b/packages/voice/src/app/agent/chat.tsx @@ -183,11 +183,10 @@ export class ChatRequest { if (!this.done) { const currentTurn = isStartConversationRequest ? value.turns.at(-1) : value; - const currentMessage = currentTurn.messages - .filter((m: any) => m.kind === 'text') - .map((m: any) => m.content) - .join(' '); + // Map non-text fragments to empty strings so that they don't appear in the transcript, + // but still introduce newlines to make clear that any earlier content is complete. + const currentMessage = currentTurn.messages.map((m: any) => (m.kind === 'text' ? m.content : '')).join('\n'); if (currentMessage === this.outMessage) { continue; } From 9fbaa1636312bb15c55fc11b5bba7798a7164e46 Mon Sep 17 00:00:00 2001 From: Peter Salas Date: Fri, 17 Nov 2023 09:46:06 -0800 Subject: [PATCH 2/6] Take a different approach --- packages/fixie-sdk/package.json | 2 +- packages/fixie-sdk/src/fixie-serve-bin.tsx | 7 ++++++- packages/fixie-sdk/src/request-wrapper.tsx | 17 +++++++++++++++++ packages/fixie-sdk/src/types.ts | 1 + packages/voice/src/app/agent/chat.tsx | 17 ++++++++++++----- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/packages/fixie-sdk/package.json b/packages/fixie-sdk/package.json index 91794581..83a449bc 100644 --- a/packages/fixie-sdk/package.json +++ b/packages/fixie-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@fixieai/sdk", - "version": "1.6.0", + "version": "1.7.0", "license": "MIT", "repository": "fixie-ai/ai-jsx", "bugs": "https://github.com/fixie-ai/ai-jsx/issues", diff --git a/packages/fixie-sdk/src/fixie-serve-bin.tsx b/packages/fixie-sdk/src/fixie-serve-bin.tsx index 1d74c1f7..8b210778 100644 --- a/packages/fixie-sdk/src/fixie-serve-bin.tsx +++ b/packages/fixie-sdk/src/fixie-serve-bin.tsx @@ -79,10 +79,15 @@ async function serve({ Readable.from( (async function* () { let lastMessages = []; + let currentValue = undefined as string | undefined; while (true) { - let currentValue = undefined as string | undefined; try { const next = await generator.next(); + if (!next.done && currentValue == next.value) { + // No change, don't send anything. + continue; + } + currentValue = next.value; lastMessages = currentValue .split('\n') diff --git a/packages/fixie-sdk/src/request-wrapper.tsx b/packages/fixie-sdk/src/request-wrapper.tsx index 0c7157c4..157024c1 100644 --- a/packages/fixie-sdk/src/request-wrapper.tsx +++ b/packages/fixie-sdk/src/request-wrapper.tsx @@ -13,6 +13,22 @@ export const RequestContext = AI.createContext<{ agentId: string; } | null>(null); +/** + * Renders to "in-progress" while the children are still being rendered, and "done" when they're done. + */ +async function* MessageState({ children }: { children: AI.Node }, { render }: AI.ComponentContext) { + const renderResult = render(children); + let didYield = false; + for await (const frame of renderResult) { + if (!didYield) { + didYield = true; + yield 'in-progress'; + } + } + + return 'done'; +} + /** * Wraps a conversational AI.JSX component to be used as a Fixie request handler. * @@ -49,6 +65,7 @@ export function FixieRequestWrapper({ {{ kind: 'text', + state: {message.element}, content: message.element, metadata: message.element.props.metadata, }} diff --git a/packages/fixie-sdk/src/types.ts b/packages/fixie-sdk/src/types.ts index 79aa6c1c..e0d6adb3 100644 --- a/packages/fixie-sdk/src/types.ts +++ b/packages/fixie-sdk/src/types.ts @@ -23,6 +23,7 @@ export interface FunctionResponseMessage extends MessageBase { export interface TextMessage extends MessageBase { kind: 'text'; content: string; + state?: 'in-progress' | 'done'; } export type Message = FunctionCallMessage | FunctionResponseMessage | TextMessage; diff --git a/packages/voice/src/app/agent/chat.tsx b/packages/voice/src/app/agent/chat.tsx index f455f629..7215b23f 100644 --- a/packages/voice/src/app/agent/chat.tsx +++ b/packages/voice/src/app/agent/chat.tsx @@ -184,11 +184,18 @@ export class ChatRequest { if (!this.done) { const currentTurn = isStartConversationRequest ? value.turns.at(-1) : value; - // Map non-text fragments to empty strings so that they don't appear in the transcript, - // but still introduce newlines to make clear that any earlier content is complete. - const currentMessage = currentTurn.messages.map((m: any) => (m.kind === 'text' ? m.content : '')).join('\n'); - if (currentMessage === this.outMessage) { - continue; + const textMessages = currentTurn.messages.filter((m: any) => m.kind === 'text'); + let currentMessage = ''; + for (const textMessage of textMessages) { + currentMessage += textMessage.content; + const messageState = textMessage.state; + if (messageState === 'in-progress') { + // This message is still being generated, so don't include any text after it. + break; + } else if (messageState === 'done') { + // Append a newline to make clear to the TTS pipeline that the text is complete. + currentMessage += '\n'; + } } // Find the longest matching prefix. From 76b232572496acdb3dbe8b8adf2d6ff85227d347 Mon Sep 17 00:00:00 2001 From: Peter Salas Date: Fri, 17 Nov 2023 11:18:00 -0800 Subject: [PATCH 3/6] Fix lint --- packages/fixie-sdk/.eslintrc.cjs | 5 ++++- packages/fixie-sdk/src/request-wrapper.tsx | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/fixie-sdk/.eslintrc.cjs b/packages/fixie-sdk/.eslintrc.cjs index 47bc6b9b..02685397 100644 --- a/packages/fixie-sdk/.eslintrc.cjs +++ b/packages/fixie-sdk/.eslintrc.cjs @@ -15,7 +15,10 @@ module.exports = { rules: { 'no-unused-vars': 'off', - '@typescript-eslint/no-unused-vars': ['warn', { ignoreRestSiblings: true, argsIgnorePattern: '^_' }], + '@typescript-eslint/no-unused-vars': [ + 'warn', + { ignoreRestSiblings: true, argsIgnorePattern: '^_', varsIgnorePattern: '^_' }, + ], 'no-undef': 'off', 'no-magic-numbers': 'off', diff --git a/packages/fixie-sdk/src/request-wrapper.tsx b/packages/fixie-sdk/src/request-wrapper.tsx index 157024c1..1a7827c0 100644 --- a/packages/fixie-sdk/src/request-wrapper.tsx +++ b/packages/fixie-sdk/src/request-wrapper.tsx @@ -19,7 +19,7 @@ export const RequestContext = AI.createContext<{ async function* MessageState({ children }: { children: AI.Node }, { render }: AI.ComponentContext) { const renderResult = render(children); let didYield = false; - for await (const frame of renderResult) { + for await (const _ of renderResult) { if (!didYield) { didYield = true; yield 'in-progress'; From 3b84ffec54a041444d706c7c6a6056ba4b2e5eb0 Mon Sep 17 00:00:00 2001 From: Peter Salas Date: Fri, 17 Nov 2023 14:21:56 -0800 Subject: [PATCH 4/6] Add frame batching --- packages/ai-jsx/package.json | 2 +- packages/ai-jsx/src/core/render.ts | 39 +++++++++++++++++++++- packages/examples/test/core/render.tsx | 34 +++++++++++++++++++ packages/fixie-sdk/package.json | 2 +- packages/fixie-sdk/src/fixie-serve-bin.tsx | 12 +++---- packages/fixie-sdk/src/request-wrapper.tsx | 8 ++++- yarn.lock | 2 +- 7 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 packages/examples/test/core/render.tsx diff --git a/packages/ai-jsx/package.json b/packages/ai-jsx/package.json index bc46c9fc..e41ebce8 100644 --- a/packages/ai-jsx/package.json +++ b/packages/ai-jsx/package.json @@ -4,7 +4,7 @@ "repository": "fixie-ai/ai-jsx", "bugs": "https://github.com/fixie-ai/ai-jsx/issues", "homepage": "https://ai-jsx.com", - "version": "0.26.0", + "version": "0.27.0", "volta": { "extends": "../../package.json" }, diff --git a/packages/ai-jsx/src/core/render.ts b/packages/ai-jsx/src/core/render.ts index 9f8ab852..124275b1 100644 --- a/packages/ai-jsx/src/core/render.ts +++ b/packages/ai-jsx/src/core/render.ts @@ -105,6 +105,12 @@ interface RenderOpts { * Indicates that the stream should be append-only. */ appendOnly?: boolean; + + /** + * Indicates that intermediate frames should be skipped if the next + * frame is available without performing I/O. + */ + batchFrames?: boolean; } /** @@ -424,8 +430,35 @@ function createRenderContextInternal( // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing const shouldStop = (opts?.stop || (() => false)) as ElementPredicate; const generatorToWrap = renderStream(context, renderable, shouldStop, Boolean(opts?.appendOnly)); + + let nextPromise = generatorToWrap.next(); while (true) { - const next = await generatorToWrap.next(); + let next = await nextPromise; + + if (!next.done && opts?.batchFrames) { + // We use `setImmediate` or `setTimeout` to ensure that all (recursively) queued microtasks + // are completed. (Promise.then handlers are queued as microtasks.) + // See https://developer.mozilla.org/en-US/docs/Web/API/HTML_DOM_API/Microtask_guide + const nullPromise = new Promise((resolve) => { + if ('setImmediate' in globalThis) { + setImmediate(() => resolve(null)); + } else { + setTimeout(() => resolve(null), 0); + } + }); + + while (!next.done) { + nextPromise = generatorToWrap.next(); + + // Consume from the generator until the null promise resolves. + const nextOrNull = await Promise.race([nextPromise, nullPromise]); + if (nextOrNull === null) { + break; + } + next = nextOrNull; + } + } + const value = opts?.stop ? (next.value as TFinal) : (next.value.join('') as TFinal); if (next.done) { if (promiseResult === null) { @@ -444,6 +477,10 @@ function createRenderContextInternal( // Otherwise yield the (string) value as-is. yield value; } + + if (!opts?.batchFrames) { + nextPromise = generatorToWrap.next(); + } } })() as AsyncGenerator; diff --git a/packages/examples/test/core/render.tsx b/packages/examples/test/core/render.tsx new file mode 100644 index 00000000..938d4bfa --- /dev/null +++ b/packages/examples/test/core/render.tsx @@ -0,0 +1,34 @@ +import * as AI from 'ai-jsx'; + +it('ensures that unbatched synchronous are not batched', async () => { + async function* MyComponent() { + yield '1'; + yield '2'; + return '3'; + } + + const ctx = AI.createRenderContext(); + const renderResult = ctx.render(, { batchFrames: false }); + const frames: string[] = []; + for await (const frame of renderResult) { + frames.push(frame); + } + expect(frames).toEqual(['1', '2']); + expect(await renderResult).toBe('3'); +}); + +it('ensures that synchronous updates are batched', async () => { + async function* MyComponent() { + yield '1'; + yield '2'; + return '3'; + } + + const ctx = AI.createRenderContext(); + const renderResult = ctx.render(, { batchFrames: true }); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _ of renderResult) { + throw new Error('Render updates should be batched'); + } + expect(await renderResult).toBe('3'); +}); diff --git a/packages/fixie-sdk/package.json b/packages/fixie-sdk/package.json index 83a449bc..1215b891 100644 --- a/packages/fixie-sdk/package.json +++ b/packages/fixie-sdk/package.json @@ -21,7 +21,7 @@ "fixie-serve-bin": "dist/fixie-serve-bin.js" }, "peerDependencies": { - "ai-jsx": ">=0.17.3 <1.0.0" + "ai-jsx": ">=0.27.0 <1.0.0" }, "dependencies": { "@opentelemetry/api": "^1.6.0", diff --git a/packages/fixie-sdk/src/fixie-serve-bin.tsx b/packages/fixie-sdk/src/fixie-serve-bin.tsx index 8b210778..3fca4db3 100644 --- a/packages/fixie-sdk/src/fixie-serve-bin.tsx +++ b/packages/fixie-sdk/src/fixie-serve-bin.tsx @@ -71,7 +71,10 @@ async function serve({ ); - const generator = createRenderContext({ enableOpenTelemetry: true }).render(renderable)[Symbol.asyncIterator](); + + // Enable frame batching to run ahead as aggressively as possible and ensure we don't have frame tearing. (See MessageState.) + const renderResult = createRenderContext({ enableOpenTelemetry: true }).render(renderable, { batchFrames: true }); + const generator = renderResult[Symbol.asyncIterator](); return res .status(200) .type('application/jsonl') @@ -79,15 +82,10 @@ async function serve({ Readable.from( (async function* () { let lastMessages = []; - let currentValue = undefined as string | undefined; while (true) { + let currentValue = undefined as string | undefined; try { const next = await generator.next(); - if (!next.done && currentValue == next.value) { - // No change, don't send anything. - continue; - } - currentValue = next.value; lastMessages = currentValue .split('\n') diff --git a/packages/fixie-sdk/src/request-wrapper.tsx b/packages/fixie-sdk/src/request-wrapper.tsx index 1a7827c0..bb06d3a0 100644 --- a/packages/fixie-sdk/src/request-wrapper.tsx +++ b/packages/fixie-sdk/src/request-wrapper.tsx @@ -14,7 +14,13 @@ export const RequestContext = AI.createContext<{ } | null>(null); /** - * Renders to "in-progress" while the children are still being rendered, and "done" when they're done. + * Renders to "in-progress" while `children` is still being rendered, and "done" when it's done. + * + * `children` should already be memoized to ensure that it's only rendered once. + * + * To ensure that this component renders consistently with `children`, a render containing both + * nodes MUST use frame batching. Without it, there will be frames where the result of this component + * will be inconsistent with the component whose rendering it's tracking. */ async function* MessageState({ children }: { children: AI.Node }, { render }: AI.ComponentContext) { const renderResult = render(children); diff --git a/yarn.lock b/yarn.lock index 3067963f..9c75ac7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3102,7 +3102,7 @@ __metadata: typescript: ^5.1.3 yargs: ^17.7.2 peerDependencies: - ai-jsx: ">=0.17.3 <1.0.0" + ai-jsx: ">=0.27.0 <1.0.0" bin: fixie-serve-bin: dist/fixie-serve-bin.js languageName: unknown From 480f4498104ef7c86a246e5da805f55ce4cffae8 Mon Sep 17 00:00:00 2001 From: Peter Salas Date: Fri, 17 Nov 2023 16:18:35 -0800 Subject: [PATCH 5/6] Use two newlines --- packages/voice/src/app/agent/chat.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/voice/src/app/agent/chat.tsx b/packages/voice/src/app/agent/chat.tsx index 7215b23f..a1b8241a 100644 --- a/packages/voice/src/app/agent/chat.tsx +++ b/packages/voice/src/app/agent/chat.tsx @@ -193,8 +193,8 @@ export class ChatRequest { // This message is still being generated, so don't include any text after it. break; } else if (messageState === 'done') { - // Append a newline to make clear to the TTS pipeline that the text is complete. - currentMessage += '\n'; + // Append two newlines to end the paragraph (i.e. make clear to the TTS pipeline that the text is complete). + currentMessage += '\n\n'; } } From cba6087594c2daa3fe2712d9f7dc910322690b27 Mon Sep 17 00:00:00 2001 From: Peter Salas Date: Fri, 17 Nov 2023 16:27:01 -0800 Subject: [PATCH 6/6] Fix test text --- packages/examples/test/core/render.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/examples/test/core/render.tsx b/packages/examples/test/core/render.tsx index 938d4bfa..44bcd71a 100644 --- a/packages/examples/test/core/render.tsx +++ b/packages/examples/test/core/render.tsx @@ -1,6 +1,6 @@ import * as AI from 'ai-jsx'; -it('ensures that unbatched synchronous are not batched', async () => { +it('ensures that synchronous updates are not batched when batchFrames is false', async () => { async function* MyComponent() { yield '1'; yield '2'; @@ -17,7 +17,7 @@ it('ensures that unbatched synchronous are not batched', async () => { expect(await renderResult).toBe('3'); }); -it('ensures that synchronous updates are batched', async () => { +it('ensures that synchronous updates are batched when batchFrames is true', async () => { async function* MyComponent() { yield '1'; yield '2';