From 0b7ceba1f7f56a0638b834ad7c0e0031b27bcdf9 Mon Sep 17 00:00:00 2001 From: Raditya Harya Date: Tue, 16 Apr 2024 19:42:43 +0700 Subject: [PATCH] feat: store image metadata in db --- bun.lockb | Bin 161362 -> 161394 bytes package.json | 1 + prisma/schema.prisma | 6 ++ src/events/message-create.ts | 20 +++- src/lib/helpers.ts | 55 ++++++++--- src/lib/llm.ts | 85 ++++++++++++----- src/lib/tracemoe.ts | 176 ++++++++++++++++++++++++++++++++++- src/utils/metadataLogger.ts | 30 ++++++ 8 files changed, 327 insertions(+), 46 deletions(-) create mode 100644 src/utils/metadataLogger.ts diff --git a/bun.lockb b/bun.lockb index 864c7b9a1567b4ae10b082296c9ec85704df8009..b46d55001ffea0e9d1316a55f7fc207880fdef63 100755 GIT binary patch delta 12501 zcmeHNd0bW1+TLpe>QPbB3XXt`N~jQnfJJh&Y*5tFOvJ)MB}74x!5mVKX%0EN)FqDD zH8nLg%}#1gnOZhn^p-{CkWE&aL)VuDrYhGvX zwb%aRwll|V(>jZz(;vR8JXaV-9m8;z78Du0dO@~9&do3EU7Rz-2rvz!Hsq3i*~PS& z?`;^jLLLXc4Llfp3wWB3VblkgmgML4rXT17AXj=BMku%)lX=JGc~bW3acDyKDIrm>J)#?SsL=kguSCA>bdtEYNcVN{}atkBTx z{QSIZl~`Gv(ss$uM|YZ&-BgM1C69(W*_1`WWh`9?${=w3(-u*r3UW!N-#@t z8q8dJX!%Pp{ci%Zc~*kyUxCpa;Z7$4R(*@|^GXv8V-Boo5D#lsXS5t@MwREogsl(; zElS{wLEo_33Xz)J7tBf*7Uc}g&NGZzc6ae7f?1p-^c{IP7^cqB;+DxQ{z$~LdLH}q zK)1gF=;`-KFiLT^Q=yNUK>=EhS7&W-JKB#0t3Y2egMz$*oS447jTa!Z8d);aE2KPG zTa}gN^=0a!nBtt$;-QB7ZW#7Zck30!3@Xbh%P~HIOs@f9ZhjXsi@5+Jh58v_mL^8~ z-4O1s*04g$y)iVqARpNn%b`cQa_0gBm>{>Q+i+l6K}<o42*h03*laWWMAl;1<^kNCB+iDGGQhl{yovly@%;pCe5{qPJkV71l6+sT4`ZWxr zo2Bg|?ZFP8-H>}&_Kj;AMy@;||sg z<*@D68o`Xs8n?>1QEBMjswjugDy$C=$l&IwwlfGlf@C#hc6f^IUJTUkkdP!W#kLGm zOGs#m$Q1E~v`0I999SjV$>7LTF-2BHJH!XFD%xT5#u`FPFBuk@Vr!)(v^CnZ3=-Q~ z$i>kqVx_EV<`CaXdvk{^4J%Ct#Gqv&Qf%Y3WXi9QYCk0QiI-d)ks|!0y@f-(A~SKn zAS+rpY!0k8_qfxdKPN-#Xj#)!6{I*whLyh!>(0H9yyfDiDWadOigDPsLC2h}rV%!o z8H;|xGStp(2Olp%V*XaCZAT!rfMk_T)RmccI&6<%8RCet#*p}%th&=7j!1i)L-@DV0WM;g>%m|U6C#BieW4&QvZB`?E4aq$)7YC)7O+sa( z#5A)#{r6Q3xh>Od9kIA^utSM4w=haJN=oxN4rLn|oRsR*3Tqxh5vl9vq&)jz z`)CYk0!yvY4B$mJ0W`ODRcDMncBOSyXCVv9bLFH-wJ=?*jMH0+_+q+U|NRbVZ z>2QhGUyt#GW314AR%*NJG3{P!p$%TU4zu&$fSv_;lkqae`i`K_JO&a9@VM;FPT|<%|XJ9zttI5 zXR5x#jZO6vn06<$U3I4FXWYmqbv&8&r?pIGpPU1;k1v85bxB~5@VcZUu274sI#cz# zjyK_zvKPH;ro$Rqz8*8GmX0UWz7Cjv>dL%vb-FO*t1YTCd+Ii=|0_&Wf8OMlasGCC zZKP9^IlUvS*ab5~tg+Ta$KHs!SR`t@>TJr(GyU5$j}#sI-*9cDYpc_dh0L0DryWY; zeoEj6z%+P>H+gxye>~V|uVXua@hoB#F;+7>c(9hqri_{GPBm0({}Ri|6a1}FF$}3# zlHr=4rBBEswOkHnp3hTx4O2Bv>#xW3J6^|=IVL7*8SGvg;Z&`!&eTuS zb~Ch{)OKV(jVP_3qxJup^{k))OEym@sLmWsOQ2`eQXNlbL6&K`I#cxuZg+t9f!T7O zYP)}9HZ3#$TpN%X{8Gzg#vj&lb!NeiX+4=4{S2n+l-83uxz2-G;0s_zUBV6T??!no zc&C9MbGe4OAl;_*jEmNwqyNj>UJnQDOE!x6*#$|)we6~d++89P}c0AixOm%(O zL_8>bUdAVudTL)2l)|r(w`)w4nCZn(ZO@0or6KC0CgNA>RJ-QFPsu3prK*e<-Y&lw z(W-BBh-aeOqZ%|_G%V+9D6e82#SN`TwXQ_#_?OgE>q@nb z?U|m4(6V@|^XeD1-=BZm|h=#4{kJm6Ag*Gwh zkJoUm^F!FP5&EpwapPy0%BG8vw8b@2N8zKAaTZlz&AF?HAHOym)1hOPomz))N5&-e z8eQBbs96%ejZ0?%93|f82$7 z2H>97AQLb5#R!xE+<)Y3*ahqXI1Anf?n4~6!wQi&5!@0Ogm5uX0+br6*BsHfJRExN zjeZaO0B~AxvzB|&s{l@kHNaY61#kjse+Iq-CP3ym=YZzKjH3_>TTEsfH#6U`#Gg^fTw|6;2YTS^B{MkoWLkx3}6Q~aSilBz#I4t z;XdH`;OR&(6gUf+dtY21#{;_&$JJ~#a2)Y`ZR2V+0_X`$fNm_nZEbGeaiebpPzjlD zGc^%r9Jj&O1D&`9c?JP4PX7eB__zQc=(&}B8Q@m+?_jPz=fM}Wj(Xah;>5+(PQ5o* zEE4@)IrGFG(S%coQ>ibI4Lk+(0eS=S@BY-ggJD9dl0iJjl zGVddlbAhnAy1gil*RTy)=C1Hu)qa_1?kZX)ntBB!LuW&-o`W-N9bRfhr3g}MSBTES zp?+T>>{yHKD@C)~qE}pOVq$Ef(Mxq&33aZ@T`3YpfSR{bbP{{i50v(+;8nsN$kqHd zIDh+mi}P#VJ@Jya&}oe;stXK6uo^(RTkTj4>aG@1Dpivq1$s`xeX_3Kw5{`gNb?r) zv2n2pDBEgvf_9b4!BnSI*sH>BdV8yTUKIg>o`Z82C$^0k`HzaSt9iz$LKr4_j^3T@ zad1|>4jcTh8h8%r1qEK1_-$hS16N}Pt1U9+m-7Y{ao5ySZ`}&IO>_%Ys`_h0VxZ^LUGG7Wt=rV8-^vud63`m)m|H4q4T|PD zp0}^Zipuu?T=Xpr*j#ZqO{k{BKnzqXNe$GlH6m9mRWWPP7M|mIgI9-@^?W1lBU2>B z#wXCN=V0DltM9wd&zv;X6v=pKLhC!r)nz0N^qkC=7A#Gku8U8CKA3REuHT{+MBzfV71vNK~|kx8-W8_Y!Y=z0u>7&UIa zXl}dT?w)9!)n+8qp2U;tB&7m%8GZvj#|JYH*mvb`8~X~1)G{`aV}7EFe;;#-Ce!7PHc{CJGAh$FarCidvW9LwcyWC@KL5TWhi+I|!hpHoQi3IcA zP}Om(2nhNl)LpmbhkyD0^qZ&RtzNL&ARtURVS&?n>ihN60UxuujS$Dc9(D_ zdrmL5I&pVJR?Ts@xJ#3OC2J47H$d%wo$_h)j~~rHdn-pvVmxxF3RfSvP?L+Wz|+3- zWTz(_xy8G)u3Gp;%zhIFu@P>Ae~ej(R=wvG3^-wuV&mFIs6%g~7!OCN&amv?8QU^>bY4Y1ihqvuIanH*OI3$y#mYl*hDtesz|kb zGs?9A76HihNYi2E@3%dZXw`&GY8 z!}1vyn~Yb5CTbNeJZDyi>?jMn=g^rQ7(Vbwi>^&o>=x9(bLzF$yhYKeUw_=ivar_L zT}@T*E%4zv?|M1Bu=95>wEgm`j}~{RweXSbIq7QF|M`>dZy(wX3%#Iw&L(Gcoo`$@ zd2z3m7RxdoFvHbHoJ^QB$F_=&=7w;UybaS4^1^K>*Pl)i_2V|tFX%td4`hJZ`LvlV z)Pe2p3?g@k*Ui5=J5O_3Yt>HSEzF}4YTY{+7Vg2>(K9$* zukR9#g*EVYBi226$K9!}f{>H(YMj(Y;v9|MHsa!kU|2 zgG(1$|8}){pXiTM$F4^EMLlmUWUgihL~Soo=}M{+CxqDOiugjT^uGEJ_QPQ;t=6AN zqB!ls|4)>^?HCVY;5w#8ofG#jd-t3Wwg5h$;`-dR@4Uzow*2gYxg`a8rTtv77ez{A vwKK$wb9o1uHHE8Qs5w@s{sCrx*TDcY%1hl<*9>&k2{zv}T~ou%o!$NgDvbZB delta 12595 zcmeHNd017|-rj2i%E?Sj3XY&~0L2kS5Ga)6P?_Oa8lu+{1q}s35a&ENgmZuzWs4i*OKD5D0S!rcjX{lLGS>OBavrf{zd|toj`M&>L5AS~0Z&<&z_L|sd z?S1~V?eJ;aGd;wN1NJKoUv zjGUb8!qG+bVo$-SY)jG&Ckgm-NrEH!Ys zy#Ad+Jn=H2XWU1@XvMw7B^WbSkc*yUx?elojs9c9p21kMg52!ftd^OBjVX|M8+~ND zPgrKXGFf2c#pfGE@)~^GzlMdhx$faSjz&F919kzA7?-TYyvzN@SuOv)MN1_%R2?h2LoDfJr63Xyx<(jER&7lBUl=%yfNm9-s2ZUAmw z$QLIj`CgWo5Y2Rss25l&m2g%@b|6p}#R9i+wGr z1Eds4rgYR#v`vQ8#=; zZhMrgB^{B8wr?QCKtf9=B+2wBr|luk&xfJ$W1V8Xtc-H{{Ro{?ejb(Vw*kwntChQe zT(XsGg;BP2_Lq>G$g&uEH1E$ePR2Hl77b;3W2gDYowBrXimhWk!{`84LuN!JiYd|_ z?X>OEI>D;VYxQJlbPC40GTQ0)5_W|LWP|3(w(pSZg@QF?Mnt0R9!&K|A)!b}qHP7F zcu454;Yrfo)ajRio#8&&ASzirC6_dHinnEDQ>X1JwDj|lj;KUiE9@ea(C6sa97ybQ z^hnc0u~JqxbBd$V-rQ+xi{0fRSkN=*IhU49c^swoL*m%@$QL6MMNMgM;S?{(bX?EM zB`ut`1ne;FJY_Mi(;;=UycwwiQfo+tRlk9~rV}Jz`C{Wl@wlvP>9lQwjx}2yBMg}y zgK@&vbDzf#F_u7L{Z^}OMJPe}I5v2qO}lg*8dWoc}R?J5*( zLk+gU)&<)dZ{J5AZ=B?>1*!QqM*#Zw)tm-qx)HNnKfo9G8#fgy1_DfEA7Bu`Gzh>RWsCsmmjf8G z`@A5l*}0UEs$(`Q&$4=$o5W~hARpri*4+C`n5MZi~ z8UAC-T0OJce**OTRQpxo!Ip{&R`i+Xqn6e%$n5{)00Vpt(C-_7>84l?oF62QP7QD| z(hmUV#jgNUbvy+6bXM~(jQ^I#VCdPBOfYW^3&Kh_!KNW(_h~^c{3|oYpcuosba67} zSz0DDJ6p^D8)oz7=<;(l&(m?hE-P>e6nJtl7HLCOW&ox2H)7s@srIXm8D|;nbXl%> zg}?%&;RRi=Dl_0qT3;R4fPI|~v|jsF$Mkzu+y9M2Vc<-71I&iFwIf-`gU<$W?=rSi z0`JhglL8W%rOLE^x7L%X->dcewBBO*3663hG z{(my7_zD-+`wb@q6Ec4?vnOz2H+`=iPiuQsrs@nX0Oy+p}3g-A;2Gewf z7mHCp4cD}yDpU14F0>0orED^~X6pU4TpiO?Q`^b(zYWYdwY0vL>^-wyS163zq2UMx zYRkqxICZLP}OAlhhsRi-K)7q-4F1$ky+ zkk#@8?R_K0v%Jwk`;i%?qvlT3%kGO>vC&2sUjBuJJToiE)fn15Z6$MHi?mE;b}TN8 zJx=TYFPQPhn}+;hkxkB>?QiXx6Lo!0>iQ;w*`DdT-l|O146Uz@8Goj>lQ~bG)-sv) zIa&t0tQ(!D4JF!8m04gu^fW!I{T66HvM=Wi}}yAf=q`mv|N?B1Wsx_nH8M@Q}vV9 zlezFNg4y6pV4AMrQX5`R3U~?^3<(q!|McK{( z;>uyJfwvxn*_XE-gxL$$gE8mVtq0+oc_6;^AdIPU>p}R|gRu2K>ehpBM{c22AAr5` ztp{QDMAZjqKGfWL5XQe+KJeXo5U%@g55im1pL-`%F<+$oweoivQ)f=`3}-QA$8xK zDn`g9yVK;XUA5&kNV&4>o-~`f@C@D?JVE+4Rpin237#_f^#W#HbMB&2{-psI)!zV3jQA}OMjuV+OrO(3EG6NF<1xFM6JVXtv}CeCutoIkjASF zhIo>`NRzb*udIvep@SZC@ z4Hq^(40v6;^AMBSaKL-sG+*l+(4A8+GQ@eP@Ycv!piP)N#s%b=B$)RZ1^lA@7HM50 z=zP^Vgt&`7EcP5s_%rZg#<-~L3&oc;c*J_eRKG72&1#>58|PYAAXT+pBqBv$mA*){ z5<}GNMIx+e4h*@#NPsgc3Ah(%2eb!L04ETo4lNRicAiMp0RjNd|2qLrx(el3EaEa& zAq03O8D z1pEOTz>}Q0z$}21oacOZ0sKeH^WJX|kEdB+U8qHr5<9FULvD4+l+1nNT<0)zq$fQCRA5Dqv1p1FPtoC3H^cnG{6*Z^$g!TBa+ zHUsN`(+GG5I0x`O$q9h#hl}D_fUA58z@=IO8{hrl)WZ6W`U&z!fDeI>fU&>~U^-9? z^aIj?{=fjBFYqXbuRG8K=m>NJc;vJS;PLk!fXCvCA@gvR2dzA$yBoM4cmUubI}aV> zfiWo07i&BirwoGsqFckOhCX&v?SIm?xe)Isep70bG1(g>y=B zt#Qq94GsYY0~x>|K%HA5-n*X?S0-0zE|3Fc10w*gQcm(bfJ4Y5AWm{`G!CgX;nkEC zBB+ly@D!+v0nQ?B4Bm!RE$a2(TwxbllrPWsvlFcH;6bbgUBwHFRzgxb6be%^2JK7Q?qXUdj-mvY^2 zwW{+9{K{1Z{6f9o;Qcsj)=~d1qo!VO%2glr*efD9#QPQBl{p-xub;obu+>i9bAl-NE|>V(M7&Q173Q23)ty zR4q55-#m>Ko~Ga6-aI!iJY8{vzS5ofKlgUIirp;YLcAXe4jvWNzC*1#Nv3$L4aOrD z{|=OEGw#XzMd6`->nplmc>V+&IAE;}!~5-F&A`V4Cw|vyGb|WL4rP`)h@!%#PHx78 zTdsP(ia_3v499G86!qWM`U6w6jfri;n!KME-m|H5=Kyo=d{eXx@O=F=Mulxb$4`S8 zh&gkI%N;zu+4!96MU^^)*h2gLr$9ox_f&IUL#Y8O7tzH~b?!^hSoI>M619U>uDsLp zDc_MJtKLrAc!wP~7w5U{E$S!wm8(&R5#s&S@yhAnNB+8D?|!TQ;&@l5RC_vjzm-gB z_Eo*pYp*o49IXEHL=d_$^Of_Mu&c7yM12vg4jn-L{A*Z)O;pFNsBVH9KpLWowqmwK zsdZaLb6Y#R^_*CIzxtd;PciYRs{1;mOcnM%>i2%^nEr|V-JIPs)}j9Rm^hBoRP{Lh z=Bixyi55z}j%8e+j#Jr5T?K`BzmQB{`$PGCFaBe!DIN*&d?r|;TD>8rh$(8z8*qO{ zz5fQ9)kpmS3Eu*$hHlZuEDcl7>=A83y`Kzb&fWHC_R$qTT3vuyhS$Ym>IHa)dOte! zdAR26#4E1;@M!Cqg%x4yN4E%0@P60m{ff{OlQBzU67dflyGbofjxB?u%M+R&IAh(Q z)xl@ORJ%7tv3V^_mAxr~L+gZlzN!4R!g*#&LFyDwrCiX246qV4_h31cNbrfYo>p1Y31G#yF9K(F~_U<-ZH=>QH6meh&2*^_dPeZo9b890~n)aj!YSp{{Hf-4doDB7OuF zXRVs{XwM}j#n&S)fQ5JRc;)!t_cdP71no?Uxralt+My=xK+Me!b*vmyYSRueHFO8u z?|^$=qe({6hj%W!9%#Qqb=!%?e+-Y?;gR@O&5(z>t@;cB<1vEqcouP};++`d@8NM9 zJT_;peQedBtLv=Ji(}thaHtP+KHoI?>yLU{9@eb69Hsh{i8j5wU)ElW$m{X#(;bgqkKz5+*3qys{k0d4r^3QL z6P@S%nD+9OeoJSi^}PrSI+eqJzOjnmCWe}^(W+vb2#)c7m%H}MpHH59^K2|Uu_zhC z`=#!&U!r!~C;np?>s7CpiSSVG_t0q%FEf7s>B<3GVz7zgXGEAfi@P#C>nT`8>=yCf zX|M8kiypM>*^T?Y(K=EWc3)pDdi4%9kA|sBdpJ7QD!NgTp*i2Gf;IYYQ#cbH{;{u>9T4;26a diff --git a/package.json b/package.json index 65bec50..ce84748 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "gpt3-tokenizer": "^1.1.5", "hono": "^4.2.4", "langchain": "^0.1.33", + "langsmith": "^0.1.14", "lodash": "^4.17.21", "openai": "^4.33.1", "pino": "^8.20.0", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index cc63f3d..934c6b5 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -35,4 +35,10 @@ model RssPooler { lastChecked DateTime lastCheckedString String? // Nullable etag String? // Nullable +} + +model AnalyzedAttachmentMetadata { + id String @id @default(uuid()) + messageId String + metadata String } \ No newline at end of file diff --git a/src/events/message-create.ts b/src/events/message-create.ts index 3635c99..1f1c4ba 100644 --- a/src/events/message-create.ts +++ b/src/events/message-create.ts @@ -67,10 +67,24 @@ async function handleThreadMessage( await channel.sendTyping(); + if (message.attachments.size > 0) { + const attachment = message.attachments.first(); + if (attachment) { + const file = await tempFile(attachment.url); + message.content = `data:${file.mimeType};base64,${file.base64}`; + } + } + + const typingInterval = setInterval(() => { + channel.sendTyping(); + }, 5000); + const completion = await createChatCompletion( - buildThreadContext(messages, message.content, client.user.id), + buildThreadContext(messages, message, client.user.id), ); + clearInterval(typingInterval); + if (completion.status !== CompletionStatus.Ok) { await handleFailedRequest( channel, @@ -157,7 +171,7 @@ async function handleDirectMessage( }, 5000); const completion = await createChatCompletion( - buildDirectMessageContext(messages, message.content, client.user.id), + buildDirectMessageContext(messages, message, client.user.id), ); clearInterval(typingInterval); @@ -252,7 +266,7 @@ async function splitSend(completion: CompletionResponse, channel: DMChannel) { if (message.trim() !== '') { await channel.sendTyping(); await new Promise((resolve) => - setTimeout(resolve, (message.length / 20) * 1000), + setTimeout(resolve, (message.length / 30) * 1000), ); await channel.send({ content: message, diff --git a/src/lib/helpers.ts b/src/lib/helpers.ts index ff2ef88..e4f1ff8 100644 --- a/src/lib/helpers.ts +++ b/src/lib/helpers.ts @@ -7,16 +7,21 @@ import { type ThreadChannel, } from 'discord.js'; import GPT3Tokenizer from 'gpt3-tokenizer'; -import type OpenAI from 'openai'; import config from '@/config'; +export type MessageContext = { + role: string; + content: string; + id: string; +}; + // TODO: inject multimodal context metadata here export function buildContext( - messages: Array, - userMessage: string, + messages: Array, + userMessage: Message, instruction?: string, -): Array { +): Array { let finalInstruction = instruction; if (!finalInstruction || finalInstruction === 'Default') { @@ -29,15 +34,19 @@ export function buildContext( finalInstruction += '.'; } + finalInstruction.replace('{{user}}', userMessage.author.username); + const systemMessageContext = { role: 'system', - content: `${finalInstruction} The current date is ${format(new Date(), 'PPP')}.`, + content: `${finalInstruction} The current date is ${format(new Date(), 'PPP')}. The latest message is form ${userMessage.author.username}.`, name: 'system', + id: 'system', }; const userMessageContext = { role: 'user', - content: userMessage, + content: userMessage.content, + id: userMessage.id, }; if (messages.length === 0) { @@ -53,14 +62,15 @@ export function buildContext( for (let i = 0; i < messages.length; i++) { const message = messages[i]; const content = message.content as string; - const encoded = tokenizer.encode(content); + const encoded = tokenizer.encode(content); tokenCount += encoded.text.length; if (tokenCount > maxTokens) { contexts.push({ role: message.role, content: content.slice(0, tokenCount - maxTokens), + id: message.id, }); break; @@ -69,6 +79,7 @@ export function buildContext( contexts.push({ role: message.role, content, + id: message.id, }); } @@ -77,10 +88,10 @@ export function buildContext( export function buildThreadContext( messages: Collection, - userMessage: string, + userMessage: Message, // eslint-disable-next-line @typescript-eslint/no-unused-vars botId: string, -): Array { +): Array { if (messages.size === 0) { return buildContext([], userMessage); } @@ -108,20 +119,28 @@ export function buildThreadContext( } const context = [ - { role: 'user', content: prompt, name: 'user' }, + { role: 'user', content: prompt, name: 'user', id: initialMessage.id }, ...messages .filter( (message) => message.type === MessageType.Default && - message.content && + (message.content || message.attachments.size > 0) && message.embeds.length === 0 && (message.mentions.members?.size ?? 0) === 0, ) .map((message) => { + if (message.attachments.size > 0) { + const attachment = message.attachments.first(); + if (attachment) { + message.content = 'data:image'; + } + } + return { role: 'function', content: message.content, name: 'someName', + id: message.id, }; }) .reverse(), @@ -132,9 +151,9 @@ export function buildThreadContext( export function buildDirectMessageContext( messages: Collection, - userMessage: string, + userMessage: Message, botId: string, -): Array { +): Array { if (messages.size === 0) { return buildContext([], userMessage); } @@ -143,14 +162,22 @@ export function buildDirectMessageContext( .filter( (message) => message.type === MessageType.Default && - message.content && + (message.content || message.attachments.size > 0) && message.embeds.length === 0 && (message.mentions.members?.size ?? 0) === 0, ) .map((message) => { + if (message.attachments.size > 0) { + const attachment = message.attachments.first(); + if (attachment) { + message.content = 'data:image'; + } + } + return { role: message.author.id === botId ? 'assistant' : 'user', content: message.content, + id: message.id, }; }) .reverse(); diff --git a/src/lib/llm.ts b/src/lib/llm.ts index 73d467e..7b80769 100644 --- a/src/lib/llm.ts +++ b/src/lib/llm.ts @@ -11,6 +11,11 @@ import { import { ChatOpenAI } from '@langchain/openai'; import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; import { getAnimeDetails, getAnimeSauce, TraceMoeResultItem } from './tracemoe'; +import { + getAnalyzedAttachmentMetadataByMessageId, + setAttachmentMetadata, +} from '@/utils/metadataLogger'; +import { MessageContext } from './helpers'; const openai = new OpenAI({ apiKey: config.openai.api_key, @@ -72,13 +77,38 @@ async function traceAnimeContext(base64Image: string) { episode: match.episode, episodes: anilistResult.data.Media.episodes, genres: anilistResult.data.Media.genres, - score: anilistResult.data.Media.averageScore, description: anilistResult.data.Media.description, - video: match.video, - image: match.image, + characters: anilistResult.data.Media.characters.edges + .slice(0, 5) + .map((edge) => { + const { name, gender, description } = edge.node; + const truncatedDescription = description + ? `${description.substring(0, 47)}...` + : 'No description available'; + return `${name.full} (Gender: ${gender}, Description: ${truncatedDescription})`; + }) + .join(', ') + .replace(/, ([^,]*)$/, ' and $1'), + nextAiringDatetime: new Date( + anilistResult.data.Media.nextAiringEpisode?.airingAt * 1000, + ).toLocaleString(), + relations: anilistResult.data.Media.relations.edges + .map( + (edge) => + edge.node.title.english || + edge.node.title.romaji || + edge.node.title.native, + ) + .join(', '), + startDate: new Date( + anilistResult.data.Media.startDate.year, + anilistResult.data.Media.startDate.month - 1, + anilistResult.data.Media.startDate.day, + ).toLocaleDateString(), + score: anilistResult.data.Media.averageScore, }; - additionalContext = `The image is from the anime titled "${anime.title}". This anime falls under the genres: ${anime.genres.join(', ')}. It has an average score of ${anime.score}. The specific scene in the image is from episode ${anime.episode} out of the total ${anime.episodes} episodes. Here is a brief description of the anime: "${anime.description}". Do note that the context provided is based on the image and may not be 100% accurate.`; + additionalContext = `The image is from the anime titled "${anime.title}". This anime falls under the genres: ${anime.genres.join(', ')}. It has an average score of ${anime.score}. The specific scene in the image is from episode ${anime.episode} out of the total ${anime.episodes} episodes. Here is a brief description of the anime: "${anime.description}". The main characters in this anime are ${anime.characters}. The next episode is scheduled to air on ${anime.nextAiringDatetime}. The anime is set to release on ${anime.startDate}. The anime has relations with the following anime or mangas: ${anime.relations}.`; } } catch (error) { console.error('Error tracing anime context:', error); @@ -92,7 +122,6 @@ async function generateImageContext(file: string) { return additionalContext; } -// TODO: Save context metadata in db and asign it to the history? async function identifyImage(file: string) { const additionalContext = await generateImageContext(file); @@ -117,30 +146,38 @@ async function identifyImage(file: string) { return response; } export async function createChatCompletion( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - messages: Array, + messages: Array, ): Promise { try { - const chatMessages = messages.map(async (message) => { - switch (message.role) { - case 'system': - return new SystemMessage(message.content); - case 'user': { - if (message.content.startsWith('data:image')) { - return new SystemMessage( - await identifyImage(message.content as string), - ); + const chatMessages = await Promise.all( + messages.map(async (message) => { + switch (message.role) { + case 'system': + return new SystemMessage(message.content); + case 'user': { + if (message.content.startsWith('data:image')) { + const analyzedAttachment = + await getAnalyzedAttachmentMetadataByMessageId(message.id); + let metadata = analyzedAttachment + ? analyzedAttachment.metadata + : null; + if (!metadata) { + metadata = await identifyImage(message.content as string); + await setAttachmentMetadata(message.id, metadata); + } + return new SystemMessage(metadata); + } + return new HumanMessage(message.content); } - return new HumanMessage(message.content); + case 'assistant': + return new AIMessage(message.content); + default: + throw new Error(`Invalid message role: ${message.role}`); } - case 'assistant': - return new AIMessage(message.content); - default: - throw new Error(`Invalid message role: ${message.role}`); - } - }); + }), + ); - const completion = await chat.invoke(await Promise.all(chatMessages)); + const completion = await chat.invoke(chatMessages); const message = completion.content; if (message) { return { diff --git a/src/lib/tracemoe.ts b/src/lib/tracemoe.ts index e0f5e73..49fb6c8 100644 --- a/src/lib/tracemoe.ts +++ b/src/lib/tracemoe.ts @@ -100,26 +100,189 @@ export async function getAnimeSauce({ }; } -export async function getAnimeDetails(anilistId: number) { +interface Title { + romaji: string; + english: string; + native: string; +} + +interface Date { + year: number; + month: number; + day: number; +} + +interface NextAiringEpisode { + airingAt: number; + timeUntilAiring: number; + episode: number; +} + +interface Tag { + name: string; + description: string; + category: string; +} + +interface Name { + full: string; + native: string; +} + +interface CharacterNode { + gender: string; + description: string | null; + name: Name; +} + +interface CharacterEdge { + node: CharacterNode; +} + +interface StudioNode { + name: string; +} + +interface StudioEdge { + node: StudioNode; +} + +interface RelationNode { + id: number; + type: string; + title: Title; +} + +interface RelationEdge { + node: RelationNode; +} + +interface Media { + id: number; + idMal: number; + title: Title; + startDate: Date; + endDate: Date; + season: string; + seasonYear: number; + format: string; + status: string; + episodes: number; + duration: number; + chapters: number | null; + volumes: number | null; + genres: string[]; + isAdult: boolean; + averageScore: number; + popularity: number; + favourites: number; + nextAiringEpisode: NextAiringEpisode; + description: string; + tags: Tag[]; + characters: { + edges: CharacterEdge[]; + }; + studios: { + edges: StudioEdge[]; + }; + relations: { + edges: RelationEdge[]; + }; +} + +interface Data { + Media: Media; +} + +export interface AnilistResponse { + data: Data; +} + +export async function getAnimeDetails( + anilistId: number, +): Promise { console.log('🚀 ~ getAnimeDetails ~ getAnimeDetails:', anilistId); - const anilistResponse = await axios.post( + const anilistResponse = (await axios.post( 'https://graphql.anilist.co', { query: ` query ($id: Int) { Media(id: $id, type: ANIME) { + id + idMal title { romaji english native } - siteUrl + startDate { + year + month + day + } + endDate { + year + month + day + } + season + seasonYear + format + status episodes + duration + chapters + volumes genres + isAdult averageScore + popularity + favourites + nextAiringEpisode { + airingAt + timeUntilAiring + episode + } description(asHtml: false) + tags { + name + description + category + } + characters { + edges { + node { + gender + description + name { + full + native + } + } + } + } + studios { + edges { + node { + name + } + } + } + relations { + edges { + node { + id + type + title { + english + native + romaji + } + } + } + } } - } + } `, variables: { id: anilistId, @@ -131,7 +294,10 @@ export async function getAnimeDetails(anilistId: number) { 'Accept-Encoding': 'gzip, deflate', //https://github.com/oven-sh/bun/issues/267#issuecomment-2044596837 }, }, - ); + )) as { + status: number; + data: AnilistResponse; + }; if (anilistResponse.status !== 200) throw new Error('Failed to get anime details'); return anilistResponse.data; diff --git a/src/utils/metadataLogger.ts b/src/utils/metadataLogger.ts new file mode 100644 index 0000000..5fa6c86 --- /dev/null +++ b/src/utils/metadataLogger.ts @@ -0,0 +1,30 @@ +import { PrismaClient } from '@prisma/client'; +// import logger from './logger'; + +const prisma = new PrismaClient(); + +export async function setAttachmentMetadata( + messageId: string, + metadata: string, +) { + console.log(`Setting metadata for messageId: ${messageId}`); + const newMetadata = await prisma.analyzedAttachmentMetadata.create({ + data: { + messageId, + metadata, + }, + }); + return newMetadata; +} + +export async function getAnalyzedAttachmentMetadataByMessageId( + messageId: string, +) { + console.log(`Finding metadata for messageId: ${messageId}`); + const metadata = await prisma.analyzedAttachmentMetadata.findFirst({ + where: { messageId }, + }); + + console.log(`metadata: ${metadata}`); + return metadata; +}