Skip to content

Commit

Permalink
Ensure non-streaming usage data from function calling is in history (#…
Browse files Browse the repository at this point in the history
…5676)

It's already yielded during streaming, but it's not being surfaced for non-streaming. Do so by manufacturing a new UsageContent for the UsageDetails and adding that to the response message that's added to the history.
  • Loading branch information
stephentoub authored Nov 20, 2024
1 parent 476a196 commit 7ebb34d
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,13 @@ public override async Task<ChatCompletion> CompleteAsync(IList<ChatMessage> chat
}
}

// If the original chat completion included usage data,
// add that into the message so it's available in the history.
if (KeepFunctionCallingMessages && response.Usage is { } usage)
{
response.Message.Contents = [.. response.Message.Contents, new UsageContent(usage)];
}

// Add the responses from the function calls into the history.
var modeAndMessages = await ProcessFunctionCallsAsync(chatMessages, options, functionCallContents, iteration, cancellationToken).ConfigureAwait(false);
if (modeAndMessages.MessagesAdded is not null)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,13 +163,25 @@ public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Paramet

int secretNumber = 42;

var response = await chatClient.CompleteAsync("What is the current secret number?", new()
List<ChatMessage> messages =
[
new(ChatRole.User, "What is the current secret number?")
];

var response = await chatClient.CompleteAsync(messages, new()
{
Tools = [AIFunctionFactory.Create(() => secretNumber, "GetSecretNumber")]
});

Assert.Single(response.Choices);
Assert.Contains(secretNumber.ToString(), response.Message.Text);

if (response.Usage is { } finalUsage)
{
UsageContent? intermediate = messages.SelectMany(m => m.Contents).OfType<UsageContent>().FirstOrDefault();
Assert.NotNull(intermediate);
Assert.True(finalUsage.TotalTokenCount > intermediate.Details.TotalTokenCount);
}
}

[ConditionalFact]
Expand Down

0 comments on commit 7ebb34d

Please sign in to comment.