diff --git a/bun.lockb b/bun.lockb index f12fa61..864c7b9 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 692968c..65bec50 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,12 @@ "prisma:studio": "prisma studio" }, "dependencies": { + "@langchain/community": "^0.0.48", + "@langchain/google-genai": "^0.0.11", + "@langchain/openai": "^0.0.28", "@prisma/client": "5.12.1", "@types/body-parser": "^1.19.5", + "@types/mime-types": "^2.1.4", "axios": "^1.6.8", "croner": "^8.0.2", "date-fns": "^3.6.0", @@ -33,12 +37,14 @@ "form-data": "^4.0.0", "gpt3-tokenizer": "^1.1.5", "hono": "^4.2.4", + "langchain": "^0.1.33", "lodash": "^4.17.21", "openai": "^4.33.1", "pino": "^8.20.0", "prisma": "^5.12.1", "rate-limiter-flexible": "^5.0.0", "rss-parser": "^3.13.0", + "sharp": "^0.33.3", "tiny-glob": "^0.2.9", "tmp-promise": "^3.0.3" }, diff --git a/src/commands/ask.ts b/src/commands/ask.ts index f157134..28d1554 100644 --- a/src/commands/ask.ts +++ b/src/commands/ask.ts @@ -6,7 +6,7 @@ import { import { createErrorEmbed } from '@/lib/embeds'; import { buildContext } from '@/lib/helpers'; -import { CompletionStatus, createChatCompletion } from '@/lib/openai'; +import { CompletionStatus, createChatCompletion } from '@/lib/llm'; export default new Command({ data: new SlashCommandBuilder() diff --git a/src/commands/chat.ts b/src/commands/chat.ts index cd10660..0a814e2 100644 --- a/src/commands/chat.ts +++ b/src/commands/chat.ts @@ -28,7 +28,7 @@ import { CompletionStatus, createChatCompletion, generateTitle, -} from '@/lib/openai'; +} from '@/lib/llm'; import { PrismaClient } from '@prisma/client'; const prisma = new PrismaClient(); diff --git a/src/commands/image.ts b/src/commands/image.ts index 5d6b69c..1bbaeed 100644 --- a/src/commands/image.ts +++ b/src/commands/image.ts @@ -6,7 +6,7 @@ import { } from 'discord.js'; import { createErrorEmbed } from '@/lib/embeds'; -import { CompletionStatus, createImage } from '@/lib/openai'; +import { CompletionStatus, createImage } from '@/lib/llm'; export default new Command({ data: new SlashCommandBuilder() diff --git a/src/commands/sauce.ts b/src/commands/sauce.ts index 31b112f..5dc9ea7 100644 --- a/src/commands/sauce.ts +++ b/src/commands/sauce.ts @@ -6,119 +6,13 @@ import { SlashCommandBuilder, } from 'discord.js'; import { createErrorEmbed } from '@/lib/embeds'; +import { + getAnimeDetails, + getAnimeSauce, + TraceMoeResultItem, +} from '@/lib/tracemoe'; +import { tempFile } from '@/utils/tempFile'; import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import FormData from 'form-data'; -import axios from 'axios'; - -type TraceMoeResultItem = { - anilist: number; - filename: string; - episode: number | null; - from: number; - to: number; - similarity: number; - video: string; - image: string; -}; - -type TraceMoeResult = { - frameCount: number; - error: string; - result: TraceMoeResultItem[]; - limit: { - limit: number; - remaining: number; - reset: number; - }; -}; - -async function downloadImage(url: string): Promise { - console.log('🚀 ~ downloadImage ~ url:', url); - let response; - try { - response = await axios.get(url, { responseType: 'arraybuffer' }); - } catch (error: any) { - if (axios.isAxiosError(error)) { - throw new Error( - `Failed to fetch the image at url: ${url}. Error: ${error.message}`, - ); - } - throw error; - } - - const buffer = response.data; - - const tempFilePath = path.join(os.tmpdir(), 'tempImage.jpg'); - try { - fs.writeFileSync(tempFilePath, buffer); - } catch (error: any) { - throw new Error( - `Failed to write the image to file at path: ${tempFilePath}. Error: ${error.message}`, - ); - } - - return tempFilePath; -} - -async function getAnimeSauce(tempFilePath: string): Promise { - console.log('🚀 ~ getAnimeSauce ~ tempFilePath:', tempFilePath); - const formData = new FormData(); - formData.append('image', fs.createReadStream(tempFilePath)); - const traceResponse = await axios.post( - 'https://api.trace.moe/search?cutBorders', - formData, - { - headers: formData.getHeaders(), - }, - ); - if (traceResponse.status !== 200) - throw new Error('Failed to get anime sauce'); - return { - ...traceResponse.data, - limit: { - limit: Number(traceResponse.headers['x-ratelimit-limit']), - remaining: Number(traceResponse.headers['x-ratelimit-remaining']), - reset: Number(traceResponse.headers['x-ratelimit-reset']), - }, - }; -} - -async function getAnimeDetails(anilistId: number) { - console.log('🚀 ~ getAnimeDetails ~ getAnimeDetails:', anilistId); - const anilistResponse = await axios.post( - 'https://graphql.anilist.co', - { - query: ` - query ($id: Int) { - Media(id: $id, type: ANIME) { - title { - romaji - english - native - } - siteUrl - episodes - genres - averageScore - } - } - `, - variables: { - id: anilistId, - }, - }, - { - headers: { - 'Content-Type': 'application/json', - }, - }, - ); - if (anilistResponse.status !== 200) - throw new Error('Failed to get anime details'); - return anilistResponse.data; -} export default new Command({ data: new SlashCommandBuilder() @@ -154,8 +48,10 @@ export default new Command({ await interaction.deferReply({ ephemeral: false }); try { - const tempFilePath = await downloadImage(input.attachment.url); - const traceResult = await getAnimeSauce(tempFilePath); + const file = await tempFile(input.attachment.url); + const traceResult = await getAnimeSauce({ + tempFilePath: file.path, + }); await interaction.editReply({ content: 'Searching for anime sauce...', }); @@ -225,7 +121,7 @@ export default new Command({ await interaction.followUp({ files: [{ attachment: match.video, name: 'video.mp4' }], }); - fs.unlinkSync(tempFilePath); + fs.unlinkSync(file.path); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { console.error(error); diff --git a/src/config.ts b/src/config.ts index 266cfda..33d7a89 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,9 @@ const config = { system_prompt: process.env.OPENAI_SYSTEM_PROMPT || 'You are a helpful assistant.', }, + google_genai: { + api_key: process.env.GOOGLE_GENAI_API_KEY, + }, }; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/events/interaction-create.ts b/src/events/interaction-create.ts index ef4d40d..7b2fa92 100644 --- a/src/events/interaction-create.ts +++ b/src/events/interaction-create.ts @@ -17,7 +17,7 @@ import { RateLimiterMemory } from 'rate-limiter-flexible'; import { createActionRow, createRegenerateButton } from '@/lib/buttons'; import { createErrorEmbed } from '@/lib/embeds'; import { buildThreadContext, isApiError } from '@/lib/helpers'; -import { CompletionStatus, createChatCompletion } from '@/lib/openai'; +import { CompletionStatus, createChatCompletion } from '@/lib/llm'; const rateLimiter = new RateLimiterMemory({ points: 3, duration: 60 }); diff --git a/src/events/message-create.ts b/src/events/message-create.ts index b52f690..3635c99 100644 --- a/src/events/message-create.ts +++ b/src/events/message-create.ts @@ -27,9 +27,10 @@ import { type CompletionResponse, CompletionStatus, createChatCompletion, -} from '@/lib/openai'; +} from '@/lib/llm'; import { PrismaClient } from '@prisma/client'; import logger from '@/utils/logger'; +import { tempFile } from '@/utils/tempFile'; const prisma = new PrismaClient(); @@ -143,10 +144,24 @@ async function handleDirectMessage( await channel.sendTyping(); + if (message.attachments.size > 0) { + const attachment = message.attachments.first(); + if (attachment) { + const file = await tempFile(attachment.url); + message.content = `data:${file.mimeType};base64,${file.base64}`; + } + } + + const typingInterval = setInterval(() => { + channel.sendTyping(); + }, 5000); + const completion = await createChatCompletion( buildDirectMessageContext(messages, message.content, client.user.id), ); + clearInterval(typingInterval); + if (completion.status !== CompletionStatus.Ok) { await handleFailedRequest( channel, @@ -182,7 +197,7 @@ export default new Event({ if ( message.author.id === client.user.id || message.type !== MessageType.Default || - !message.content || + (!message.content && !message.attachments.size) || !isEmpty(message.embeds) || !isEmpty(message.mentions.members) ) { diff --git a/src/lib/helpers.ts b/src/lib/helpers.ts index 7fbb4e6..ff2ef88 100644 --- a/src/lib/helpers.ts +++ b/src/lib/helpers.ts @@ -11,6 +11,7 @@ import type OpenAI from 'openai'; import config from '@/config'; +// TODO: inject multimodal context metadata here export function buildContext( messages: Array, userMessage: string, diff --git a/src/lib/llm.ts b/src/lib/llm.ts new file mode 100644 index 0000000..73d467e --- /dev/null +++ b/src/lib/llm.ts @@ -0,0 +1,290 @@ +import axios from 'axios'; +import { truncate } from 'lodash'; +import OpenAI from 'openai'; +import logger from '@/utils/logger'; +import config from '@/config'; +import { + HumanMessage, + SystemMessage, + AIMessage, +} from '@langchain/core/messages'; +import { ChatOpenAI } from '@langchain/openai'; +import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; +import { getAnimeDetails, getAnimeSauce, TraceMoeResultItem } from './tracemoe'; + +const openai = new OpenAI({ + apiKey: config.openai.api_key, + baseURL: config.openai.base_url, +}); + +const chat = new ChatOpenAI({ + apiKey: config.openai.api_key, + model: config.openai.model, + configuration: { + baseURL: config.openai.base_url, + }, +}); + +const vision = new ChatGoogleGenerativeAI({ + model: 'gemini-pro-vision', + maxOutputTokens: 2048, + apiKey: config.google_genai.api_key, +}); + +export enum CompletionStatus { + Ok = 0, + Moderated = 1, + ContextLengthExceeded = 2, + InvalidRequest = 3, + UnexpectedError = 4, +} + +export interface CompletionResponse { + status: CompletionStatus; + message: string; +} + +async function traceAnimeContext(base64Image: string) { + let additionalContext = ''; + + try { + const traceResult = await getAnimeSauce({ + base64Image: base64Image, + }); + + const match = traceResult.result.reduce( + (prev: TraceMoeResultItem | null, current: TraceMoeResultItem) => { + if (!prev || current.similarity > prev.similarity) { + return current; + } + return prev; + }, + null, + ); + + if (match) { + const anilistResult = await getAnimeDetails(match.anilist); + const anime = { + title: + anilistResult.data.Media.title.english || + anilistResult.data.Media.title.romaji || + anilistResult.data.Media.title.native, + episode: match.episode, + episodes: anilistResult.data.Media.episodes, + genres: anilistResult.data.Media.genres, + score: anilistResult.data.Media.averageScore, + description: anilistResult.data.Media.description, + video: match.video, + image: match.image, + }; + + additionalContext = `The image is from the anime titled "${anime.title}". This anime falls under the genres: ${anime.genres.join(', ')}. It has an average score of ${anime.score}. The specific scene in the image is from episode ${anime.episode} out of the total ${anime.episodes} episodes. Here is a brief description of the anime: "${anime.description}". Do note that the context provided is based on the image and may not be 100% accurate.`; + } + } catch (error) { + console.error('Error tracing anime context:', error); + } + + return additionalContext; +} + +async function generateImageContext(file: string) { + const additionalContext = await traceAnimeContext(file); + return additionalContext; +} + +// TODO: Save context metadata in db and asign it to the history? +async function identifyImage(file: string) { + const additionalContext = await generateImageContext(file); + + const prompt = `You received an image. Describe the image in detail and extract any useful information from it.`; + + const input = [ + new HumanMessage({ + content: [ + { + type: 'text', + text: prompt, + }, + { + type: 'image_url', + image_url: file, + }, + ], + }), + ]; + const res = await vision.invoke(input); + const response = `The image you received in the discord chat has the following description: ${res.content}. Based on the additional context, it appears that ${additionalContext}. Please provide the user with any relevant information and share your thoughts on the image. If the additional context does not align with the description, please disregard it. If it does align, please provide as much context as possible.`; + return response; +} +export async function createChatCompletion( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messages: Array, +): Promise { + try { + const chatMessages = messages.map(async (message) => { + switch (message.role) { + case 'system': + return new SystemMessage(message.content); + case 'user': { + if (message.content.startsWith('data:image')) { + return new SystemMessage( + await identifyImage(message.content as string), + ); + } + return new HumanMessage(message.content); + } + case 'assistant': + return new AIMessage(message.content); + default: + throw new Error(`Invalid message role: ${message.role}`); + } + }); + + const completion = await chat.invoke(await Promise.all(chatMessages)); + const message = completion.content; + if (message) { + return { + status: CompletionStatus.Ok, + message: truncate(message.toString(), { length: 2000 }), + }; + } + } catch (err) { + logger.error(err, 'Error while processing chat completion'); + return { + status: CompletionStatus.UnexpectedError, + message: err instanceof Error ? err.message : (err as string), + }; + } + return { + status: CompletionStatus.UnexpectedError, + message: 'There was an unexpected error while processing your request.', + }; +} + +export async function createImage(prompt: string): Promise { + try { + const moderation = await openai.moderations.create({ + input: prompt, + }); + + const result = moderation.results[0]; + + if (result.flagged) { + return { + status: CompletionStatus.Moderated, + message: 'Your prompt has been blocked by moderation.', + }; + } + + const image = await openai.images.generate({ + prompt, + }); + + const imageUrl = image.data[0].url; + + if (imageUrl) { + return { + status: CompletionStatus.Ok, + message: imageUrl, + }; + } + } catch (err) { + if (axios.isAxiosError(err)) { + const error = err.response?.data?.error; + + if (error && error.code === 'context_length_exceeded') { + return { + status: CompletionStatus.ContextLengthExceeded, + message: + 'The request has exceeded the token limit. Try again with a shorter message or start another conversation.', + }; + } else if (error && error.type === 'invalid_request_error') { + logError(err); + + return { + status: CompletionStatus.InvalidRequest, + message: error.message, + }; + } + } else { + logError(err); + + return { + status: CompletionStatus.UnexpectedError, + message: err instanceof Error ? err.message : (err as string), + }; + } + } + + return { + status: CompletionStatus.UnexpectedError, + message: 'There was an unexpected error processing your request.', + }; +} + +export async function generateTitle( + userMessage: string, + botMessage: string, +): Promise { + const messages = [ + { + role: 'system', + content: 'You are a helpful assistant.', + }, + { + role: 'user', + content: userMessage, + }, + { + role: 'assistant', + content: botMessage, + }, + { + role: 'user', + content: 'Create a title for our conversation in 6 words or less.', + }, + ] as OpenAI.Chat.ChatCompletionMessageParam[]; + + try { + const completion = await openai.chat.completions.create({ + messages, + model: config.openai.model, + temperature: 0.5, + }); + + const message = completion.choices[0].message; + + if (message) { + let title = message.content?.trim(); + + if (title?.startsWith('"') && title.endsWith('"')) { + title = title.slice(1, -1); + } + + while (title?.endsWith('.')) { + title = title.slice(0, -1); + } + + return title || ''; + } + } catch (err) { + logError(err); + } + + return ''; +} + +function logError(err: unknown): void { + if (axios.isAxiosError(err)) { + if (err.response) { + logger.error( + { status: err.response.status, data: err.response.data }, + 'Axios error with response', + ); + } else { + logger.error(err.message, 'Axios error without response'); + } + } else { + logger.error(err, 'Unknown error'); + } +} diff --git a/src/lib/openai.ts b/src/lib/openai.ts deleted file mode 100644 index 67aee81..0000000 --- a/src/lib/openai.ts +++ /dev/null @@ -1,205 +0,0 @@ -import axios from 'axios'; -import { truncate } from 'lodash'; -import OpenAI from 'openai'; -import logger from '@/utils/logger'; -import config from '@/config'; - -const openai = new OpenAI({ - apiKey: config.openai.api_key, - baseURL: config.openai.base_url, -}); - -export enum CompletionStatus { - Ok = 0, - Moderated = 1, - ContextLengthExceeded = 2, - InvalidRequest = 3, - UnexpectedError = 4, -} - -export interface CompletionResponse { - status: CompletionStatus; - message: string; -} - -export async function createChatCompletion( - messages: Array, -): Promise { - try { - logger.info('🤖 Generating response'); - const completion = await openai.chat.completions.create({ - messages, - model: config.openai.model, - temperature: Number(config.openai.temperature), - max_tokens: Number(config.openai.max_tokens), - }); - - const message = completion.choices[0].message; - - if (message) { - return { - status: CompletionStatus.Ok, - message: truncate(message.content?.trim(), { length: 2000 }), - }; - } - } catch (err) { - if (axios.isAxiosError(err)) { - const error = err.response?.data?.error; - - if (error && error.code === 'context_length_exceeded') { - return { - status: CompletionStatus.ContextLengthExceeded, - message: - 'The request has exceeded the context limit. Try again with a shorter message or start another conversation.', - }; - } else if (error && error.type === 'invalid_request_error') { - logError(err); - - return { - status: CompletionStatus.InvalidRequest, - message: error.message, - }; - } - } else { - logError(err); - - return { - status: CompletionStatus.UnexpectedError, - message: err instanceof Error ? err.message : (err as string), - }; - } - } - - return { - status: CompletionStatus.UnexpectedError, - message: 'There was an unexpected error while processing your request.', - }; -} - -export async function createImage(prompt: string): Promise { - try { - const moderation = await openai.moderations.create({ - input: prompt, - }); - - const result = moderation.results[0]; - - if (result.flagged) { - return { - status: CompletionStatus.Moderated, - message: 'Your prompt has been blocked by moderation.', - }; - } - - const image = await openai.images.generate({ - prompt, - }); - - const imageUrl = image.data[0].url; - - if (imageUrl) { - return { - status: CompletionStatus.Ok, - message: imageUrl, - }; - } - } catch (err) { - if (axios.isAxiosError(err)) { - const error = err.response?.data?.error; - - if (error && error.code === 'context_length_exceeded') { - return { - status: CompletionStatus.ContextLengthExceeded, - message: - 'The request has exceeded the token limit. Try again with a shorter message or start another conversation.', - }; - } else if (error && error.type === 'invalid_request_error') { - logError(err); - - return { - status: CompletionStatus.InvalidRequest, - message: error.message, - }; - } - } else { - logError(err); - - return { - status: CompletionStatus.UnexpectedError, - message: err instanceof Error ? err.message : (err as string), - }; - } - } - - return { - status: CompletionStatus.UnexpectedError, - message: 'There was an unexpected error processing your request.', - }; -} - -export async function generateTitle( - userMessage: string, - botMessage: string, -): Promise { - const messages = [ - { - role: 'system', - content: 'You are a helpful assistant.', - }, - { - role: 'user', - content: userMessage, - }, - { - role: 'assistant', - content: botMessage, - }, - { - role: 'user', - content: 'Create a title for our conversation in 6 words or less.', - }, - ] as OpenAI.Chat.ChatCompletionMessageParam[]; - - try { - const completion = await openai.chat.completions.create({ - messages, - model: config.openai.model, - temperature: 0.5, - }); - - const message = completion.choices[0].message; - - if (message) { - let title = message.content?.trim(); - - if (title?.startsWith('"') && title.endsWith('"')) { - title = title.slice(1, -1); - } - - while (title?.endsWith('.')) { - title = title.slice(0, -1); - } - - return title || ''; - } - } catch (err) { - logError(err); - } - - return ''; -} - -function logError(err: unknown): void { - if (axios.isAxiosError(err)) { - if (err.response) { - logger.error( - { status: err.response.status, data: err.response.data }, - 'Axios error with response', - ); - } else { - logger.error(err.message, 'Axios error without response'); - } - } else { - logger.error(err, 'Unknown error'); - } -} diff --git a/src/lib/tracemoe.ts b/src/lib/tracemoe.ts new file mode 100644 index 0000000..e0f5e73 --- /dev/null +++ b/src/lib/tracemoe.ts @@ -0,0 +1,138 @@ +import fs from 'fs'; +import FormData from 'form-data'; +import axios from 'axios'; +import sharp from 'sharp'; + +export type TraceMoeResultItem = { + anilist: number; + filename: string; + episode: number | null; + from: number; + to: number; + similarity: number; + video: string; + image: string; +}; + +export type TraceMoeResult = { + frameCount: number; + error: string; + result: TraceMoeResultItem[]; + limit: { + limit: number; + remaining: number; + reset: number; + }; +}; + +async function processImage(imageSource: Buffer | string) { + return await sharp(imageSource).resize({ width: 500 }).jpeg().toBuffer(); +} + +function appendImageToFormData(formData: FormData, imageBuffer: Buffer) { + formData.append('image', imageBuffer, { + filename: 'blob', + contentType: 'image/jpeg', + }); +} +export async function getAnimeSauce({ + tempFilePath, + base64Image, +}: { + tempFilePath?: string; + base64Image?: string; +}): Promise { + const formData = new FormData(); + let imageBuffer: Buffer; + + if (base64Image) { + const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ''); + imageBuffer = Buffer.from(base64Data, 'base64'); + } else if (tempFilePath) { + imageBuffer = fs.readFileSync(tempFilePath); + } else { + throw new Error('Either a file path or a base64 string must be provided'); + } + + try { + const resizedBuffer = await processImage(imageBuffer); + appendImageToFormData(formData, resizedBuffer); + } catch (err) { + console.error('Error processing image data:', err); + console.error('Base64 string:', base64Image); + } + + const traceResponse = (await axios.post( + 'https://api.trace.moe/search?cutBorders', + formData, + { + headers: { + ...formData.getHeaders(), + 'Accept-Encoding': 'gzip, deflate', + }, + }, + )) as { + status: number; + data: TraceMoeResult; + headers: Record; + }; + + if (traceResponse.status !== 200) { + console.error(traceResponse.data); + throw new Error('Failed to get anime sauce'); + } + + if (tempFilePath) { + fs.unlink(tempFilePath, (err) => { + if (err) { + console.error('Failed to delete temp file:', err); + } + }); + } + + return { + ...traceResponse.data, + limit: { + limit: Number(traceResponse.headers['x-ratelimit-limit']), + remaining: Number(traceResponse.headers['x-ratelimit-remaining']), + reset: Number(traceResponse.headers['x-ratelimit-reset']), + }, + }; +} + +export async function getAnimeDetails(anilistId: number) { + console.log('🚀 ~ getAnimeDetails ~ getAnimeDetails:', anilistId); + const anilistResponse = await axios.post( + 'https://graphql.anilist.co', + { + query: ` + query ($id: Int) { + Media(id: $id, type: ANIME) { + title { + romaji + english + native + } + siteUrl + episodes + genres + averageScore + description(asHtml: false) + } + } + `, + variables: { + id: anilistId, + }, + }, + { + headers: { + 'Content-Type': 'application/json', + 'Accept-Encoding': 'gzip, deflate', //https://github.com/oven-sh/bun/issues/267#issuecomment-2044596837 + }, + }, + ); + if (anilistResponse.status !== 200) + throw new Error('Failed to get anime details'); + return anilistResponse.data; +} diff --git a/src/utils/tempFile.ts b/src/utils/tempFile.ts new file mode 100644 index 0000000..df18cd7 --- /dev/null +++ b/src/utils/tempFile.ts @@ -0,0 +1,57 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import axios from 'axios'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import mime from 'mime-types'; +import { v4 as uuidv4 } from 'uuid'; + +/** + * Downloads file from the specified URL and saves it as a temporary file. + * @param url - The URL of the image to download. + * @returns A promise that resolves to the path of the temporary file, the MIME type, the buffer, and the base64 string. + * @throws If there is an error while fetching or saving the image. + */ +export async function tempFile( + url: string, +): Promise<{ path: string; mimeType: string; buffer: Buffer; base64: string }> { + let response; + try { + response = await axios.get(url, { responseType: 'arraybuffer' }); + } catch (error) { + if (axios.isAxiosError(error)) { + throw new Error( + `Failed to fetch the file at url: ${url}. Error: ${error.message}`, + ); + } + throw error; + } + + const buffer = response.data; + const mimeType = response.headers['content-type']; + const extension = mime.extension(mimeType); + + if (!extension) { + throw new Error(`Unsupported MIME type: ${mimeType}`); + } + + const filename = `${uuidv4()}.${extension}`; + const tempFilePath = path.join(os.tmpdir(), filename); + + try { + fs.writeFileSync(tempFilePath, buffer); + } catch (error: any) { + throw new Error( + `Failed to write the file at path: ${tempFilePath}. Error: ${error.message}`, + ); + } + + const base64 = buffer.toString('base64'); + + return { + path: tempFilePath, + mimeType, + buffer, + base64, + }; +}