diff --git a/server/claude.js b/server/claude.js index e850bb87..1c414a3f 100644 --- a/server/claude.js +++ b/server/claude.js @@ -55,47 +55,39 @@ export const getTextClaude = async (prompt, temperature, imageBase64, fileType, }, ]; - let response = await anthropic.beta.tools.messages.create({ - model, - max_tokens: 4096, - temperature: temperature || 0.5, - tools: webTools ? tools : [], - messages, - }); + let response = await getResponse(); + let toolUses, toolResults; - if (!response) { - throw new Error("Claude Error"); - } else { - let toolUses, toolResults; - - while (response.stop_reason === "tool_use") { - toolUses = response.content.filter((block) => block.type === "tool_use"); - if (!toolUses.length) { - return response?.content?.[0]?.text; - } + while (response?.stop_reason === "tool_use") { + toolUses = response.content.filter((block) => block.type === "tool_use"); + if (!toolUses.length) { + return response?.content?.[0]?.text; + } - toolResults = await Promise.all( - toolUses.map(async (toolUse) => { - const toolResult = await handleToolCall(toolUse.name, toolUse.input, userId); - return { tool_use_id: toolUse.id, content: toolResult }; - }) - ); + toolResults = await Promise.all( + toolUses.map(async (toolUse) => { + const toolResult = await handleToolCall(toolUse.name, toolUse.input, userId); + return { tool_use_id: toolUse.id, content: toolResult }; + }) + ); - messages.push({ role: "assistant", content: response.content.filter((c) => c.type !== "text" || c.text) }); - messages.push({ - role: "user", - content: toolResults.map((toolResult) => ({ type: "tool_result", ...toolResult })), - }); + messages.push({ role: "assistant", content: response.content.filter((c) => c.type !== "text" || c.text) }); + messages.push({ + role: "user", + content: toolResults.map((toolResult) => ({ type: "tool_result", ...toolResult })), + }); + response = await getResponse(); + } - response = await anthropic.beta.tools.messages.create({ - model, - max_tokens: 4096, - temperature: temperature || 0.5, - tools: webTools ? tools : [], - messages, - }); - } + return response?.content?.[0]?.text; - return response?.content?.[0]?.text; + async function getResponse() { + return anthropic.beta.tools.messages.create({ + model, + max_tokens: 4096, + temperature: temperature || 0.5, + tools: webTools ? tools : [], + messages, + }); } }; diff --git a/server/gemini.js b/server/gemini.js index a84be59c..754bdb34 100644 --- a/server/gemini.js +++ b/server/gemini.js @@ -50,6 +50,20 @@ export async function getTextGemini(prompt, temperature, imageBase64, fileType, data: imageBase64, }, }); + } else if (fileType === "ogg") { + parts.push({ + inlineData: { + mimeType: "audio/ogg", + data: imageBase64, + }, + }); + } else if (fileType === "wav") { + parts.push({ + inlineData: { + mimeType: "audio/wav", + data: imageBase64, + }, + }); } const contents = { diff --git a/server/openai.js b/server/openai.js index 6f495a0a..8036c552 100644 --- a/server/openai.js +++ b/server/openai.js @@ -18,7 +18,7 @@ export const getTextGpt = async (prompt, temperature, userId, model, apiKey, web const openAiTools = tools.map(renameProperty).map((f) => ({ type: "function", function: f })); const messages = [{ role: "user", content: prompt }]; - const getMessage = async () => { + const getResponse = async () => { const completion = await openai.chat.completions.create({ model: model || "gpt-3.5-turbo", max_tokens: 3000, @@ -29,10 +29,10 @@ export const getTextGpt = async (prompt, temperature, userId, model, apiKey, web return completion?.choices?.[0]?.message; }; - let responseMessage = await getMessage(); - while (responseMessage?.tool_calls) { - const toolCalls = responseMessage?.tool_calls; - messages.push(responseMessage); + let response = await getResponse(); + while (response?.tool_calls) { + const toolCalls = response?.tool_calls; + messages.push(response); for (const toolCall of toolCalls) { const toolResult = await handleToolCall( toolCall.function.name, @@ -46,7 +46,7 @@ export const getTextGpt = async (prompt, temperature, userId, model, apiKey, web content: toolResult, }); } - responseMessage = await getMessage(); + response = await getResponse(); } - return responseMessage?.content; + return response?.content; }; diff --git a/src/components/ChatHistory.js b/src/components/ChatHistory.js index 7d51b875..de6f4174 100644 --- a/src/components/ChatHistory.js +++ b/src/components/ChatHistory.js @@ -23,6 +23,8 @@ const getFileTypeIcon = (mimeType) => { case "jpg": return null; case "mp3": + case "ogg": + case "wav": case "mpeg": case "x-m4a": return "🔊"; diff --git a/src/components/FileSelector.js b/src/components/FileSelector.js index f291453e..9f7a8800 100644 --- a/src/components/FileSelector.js +++ b/src/components/FileSelector.js @@ -16,7 +16,7 @@ const FileSelector = ({ onFileSelect, selectedFile, allowedFileTypes }) => { if (type === "image") { acc.push(".png", ".jpeg", ".jpg"); } else if (type === "audio") { - acc.push(".mp3", ".m4a"); + acc.push(".mp3", ".m4a", ".ogg", ".wav"); } else if (type === "video") { acc.push(".mp4"); } else if (type === "document") { @@ -25,7 +25,7 @@ const FileSelector = ({ onFileSelect, selectedFile, allowedFileTypes }) => { return acc; }, []) .join(",") - : ".pdf,.doc,.docx,.xls,.xlsx,.png,.jpeg,.jpg,.mp4,.mp3,.m4a"; + : ".pdf,.doc,.docx,.xls,.xlsx,.png,.jpeg,.jpg,.mp4,.mp3,.m4a,.ogg"; return (