fix(anthropic): Add cache usage metadata to Anthropic streaming respo…

…nses (#7368)
langchain-ai · Dec 15, 2024 · f05af16 · f05af16
1 parent 2cdf57c
commit f05af16
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/libs/langchain-anthropic/src/tests/chat_models.int.test.ts b/libs/langchain-anthropic/src/tests/chat_models.int.test.ts
@@ -721,6 +721,16 @@ test("system prompt caching", async () => {
   expect(res2.response_metadata.usage.cache_read_input_tokens).toBeGreaterThan(
     0
   );
+  const stream = await model.stream(messages);
+  let agg;
+  for await (const chunk of stream) {
+    agg = agg === undefined ? chunk : concat(agg, chunk);
+  }
+  expect(agg).toBeDefined();
+  expect(agg!.response_metadata.usage.cache_creation_input_tokens).toBe(0);
+  expect(agg!.response_metadata.usage.cache_read_input_tokens).toBeGreaterThan(
+    0
+  );
 });
 
 // TODO: Add proper test with long tool content

diff --git a/libs/langchain-anthropic/src/utils/message_outputs.ts b/libs/langchain-anthropic/src/utils/message_outputs.ts
@@ -30,16 +30,22 @@ export function _makeMessageChunkFromAnthropicEvent(
         filteredAdditionalKwargs[key] = value;
       }
     }
+    const { input_tokens, output_tokens, ...rest } = usage ?? {};
     const usageMetadata: UsageMetadata = {
-      input_tokens: usage.input_tokens,
-      output_tokens: usage.output_tokens,
-      total_tokens: usage.input_tokens + usage.output_tokens,
+      input_tokens,
+      output_tokens,
+      total_tokens: input_tokens + output_tokens,
     };
     return {
       chunk: new AIMessageChunk({
         content: fields.coerceContentToString ? "" : [],
         additional_kwargs: filteredAdditionalKwargs,
         usage_metadata: fields.streamUsage ? usageMetadata : undefined,
+        response_metadata: {
+          usage: {
+            ...rest,
+          },
+        },
         id: data.message.id,
       }),
     };