Revert "Request less max_new_tokens on tools and reduce truncate …

…on CMD R+ (#1292)" This reverts commit c1fc4f9.
huggingface · Jun 18, 2024 · a547bbf · a547bbf
1 parent c1fc4f9
commit a547bbf
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/chart/env/prod.yaml b/chart/env/prod.yaml
@@ -47,7 +47,7 @@ envVars:
         "tools": true,
         "parameters": {
           "stop": ["<|END_OF_TURN_TOKEN|>"],
-          "truncate" : 12288,
+          "truncate" : 28672,
           "max_new_tokens" : 2048,
           "temperature" : 0.3
         },

diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts
@@ -131,7 +131,7 @@ export async function* runTools(
 	for await (const output of await endpoint({
 		messages: messagesWithFilesPrompt,
 		preprompt,
-		generateSettings: { ...assistant?.generateSettings, max_new_tokens: 512 },
+		generateSettings: assistant?.generateSettings,
 		tools,
 	})) {
 		// model natively supports tool calls