perf: Fine-tune max tokens for lower latency conversation

microsoft · Feb 12, 2024 · 30ff579 · 30ff579
1 parent dc65da6
commit 30ff579
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/main.py b/main.py
@@ -1020,7 +1020,7 @@ async def _error_response() -> Tuple[CallModel, ActionModel]:
     tool_calls = {}
     try:
         async for delta in completion_stream(
-            max_tokens=400,
+            max_tokens=350,
             messages=messages,
             tools=tools,
         ):