From f6d91ae9ffa4567a5f3fde52b7060819d0279c45 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Mon, 20 Mar 2023 23:28:59 +0800 Subject: [PATCH 01/28] [script] update warehouse jpa store metrics expire-time to 1h --- home/src/constants.js | 5 +++++ home/src/pages/index.js | 3 ++- home/static/img/icons/bugstack_logo.png | Bin 0 -> 21585 bytes script/application.yml | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 home/static/img/icons/bugstack_logo.png diff --git a/home/src/constants.js b/home/src/constants.js index 0d707d28b77..5201b6cbbd8 100644 --- a/home/src/constants.js +++ b/home/src/constants.js @@ -222,6 +222,11 @@ export const friendsLink = [ alt: 'hippo4j', url: 'https://hippo4j.cn/', }, + { + img: 'bugstack_logo.png', + alt: 'bugStack', + url: 'https://bugstack.cn/', + } ] export const usersLink = [ diff --git a/home/src/pages/index.js b/home/src/pages/index.js index ed4f09b5e98..6b27ea35c6a 100644 --- a/home/src/pages/index.js +++ b/home/src/pages/index.js @@ -252,7 +252,8 @@ function autoRedirect() { let lang = global.navigator?.language || navigator?.userLanguage console.log('Current lang is ' + lang) if (lang != null && (lang.toLowerCase() === 'zh-cn' || lang.toLowerCase().indexOf('zh') > 0)) { - if (sessionStorage.getItem('auto_detect_redirect') !== 'true') { + console.log(window.location.pathname); + if (sessionStorage.getItem('auto_detect_redirect') !== 'true' && !window.location.pathname.startsWith('/zh-cn', false)) { console.log('current lang is zh-cn, redirect to zh-cn') sessionStorage.setItem('auto_detect_redirect', 'true') window.location.href = '/zh-cn' diff --git a/home/static/img/icons/bugstack_logo.png b/home/static/img/icons/bugstack_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..addf6e2132b6f3589c86c436e36be63f12b83b61 GIT binary patch literal 21585 zcmcHgWmsF!`UVQ)THK{T(ctc0q-Y_y6em#J-QA%;(Bkf{#fn2HMT$G6xVyti_ujw# z-|u@qpLJc6nXEOk=9&BPBue$;2Mjb)G#D5d40$J{Of@P~U!OhQ#d!le2!rp!$(6D4Exq&FOOf}2;L zo46-0hrMtyB&Cku<1~e_PETtoO1(--PGaNOJ;Nqq)6YLzx#?X|B%DUDpNdjo?Om2< z5sAP=hl5H|f2XC-OK#}G&2h`~uo@lpaN%>+9f#|?>odVfkkSkvC9Y7xPkum*qG zS)=ju_2zcKm`~fHh*E=Vs|2yFZvc`yh)iZs2 zmgSt=N%ipJnvIQ+>F5y2sf*pRNL%hh`A}V;3@syF(K|CF^~j`%0E?1kZdCC1dJMd; zlZg>Bzc`%*&_;0B`(dO|Buz<7DkAT$Pbq~A%aNqlVLWEfo6I(xDN#fO*6vU_nvg%O zIL>MWh#gvcLdbC1`@01uU(J_m@d9sQK5N=_=r$hDF&+mX)WoS!#+q}5ipGKg<|GmCI%il5o zuHloyVUVHUaG}LB5AL7daF9Isf7&p%&}T4`>QeIZ&|2Nh83eL-`Rw4TzRkY_y@Bc| zr|SX(LrC|xz{;!9onN*INjW%!sQB18*f^-g(5R@WM4Zhngw&*E z|1%u=O_chxtE;0BJG+O62b%{sn}f3@JEx$aAUg*aI~Nx#v)-DQ^0fNDmF!*qvn}Wb+5bLa=Vaqx z|L@$;p(20p3aMIof^2oAt?ZyQgRUXQ$sr)}um1n9C;zwN|BTf6zmfc$JpViLf1dpR zkJNMlIZHX%L6>wD`@iG*&*1-k_@99y?056T|CxL&p8erT{ z|LY_>dvTR^*O&6gQP*zjy=BAWMEh<|@}|+dfj$%txL6pP@BdegVvl^679^o~3c@i) zAcmC<{HL14HZJ&q1^2&wp(Bk0Vemlks%??~bo*ye!H-CaE@g?|L5YOhqHfls;Jfr9s`X0vKRY^;GM*ekZ8Bc$@TVIpjXRjl z3+fUk(=Pcvoh<&WR3Cy@ZnLaizNem0{F_?U@!3UX@m>3&RAU)jpaT2jf-=oN%sF2O+dF#oUQmj73p!24Frz7Xfj(^pF?8rQko4ZNLOwDS%DNZPz%%{l#4PNUT8$Vv4BnKI-vD9<pVT9Fr{>ZMi->7z}CZF=eQ{oiKyo5c)%RJ={$ z2D&4V@tgjG7JI5M6j2wgkC>`sZ1sJw*Yji&ciF2c%nWf9Oq z<`URAK51j5dej zAtF1N+|veG>;6Z}pE+qr#p;zMGv#hYP2CyPcpz@`wyb{;^F^Af3mep?tKm}| zR!z5v(vr1!S_BMcNnjdI4tugNMxEONO)62BoL)R3k#!7o`ZT` zUutZq=j=||s>{8$yu0Azi?+c;QpK!J5$kCroH-g$)Lx=8r8Oa?+eaGQJ5=>TEL zxaI%K4u6y){(_^9t6TB4_saPeJROJvS3)q!8KXg zd?3!%g!rq12|^4P%UB&pWi%^L>~*a6cePWrYtMErJ5HT=B9K*mC{`Z&{tS|v5P3aX z2n>?$y=%TQ*DrE^F13Jkwpndc2n$DULYk!R<$8WSGxw&!V~(-#%>|eH=XF|} zTo$1JQ)0y4eqRD}0#PgyidBonv+Q!dKzswIv^!&!Uv)v4=KZQ0<)auW1!Wt1Ph{Kl zaD6Tp9!N^3{mWnM1Wi&yWpC46Wqi(BR0CFRm_u#JyQt)%=MR3z5}u{BL+UNdjY%v( zOw0b29EV%rbJ@@CF>X;VAYZs_DxhCmtN^6VN*xYV!C}`f)iIGC$ZnTUMdAEW$mNDb4p2a(6 zmkOj`p&&>^6++j3lak(=F69=a`1^|vJ%nxjOVfe(W(aDO+(Fk8V@9sXdmPN;^$>{Y zYnICRd16$9*)B~yI%3N`BbP31b}a`>X_ypZ z^t@X6!HA!4iVz_FyKj}i#n{Q(3Ww4pa<)q?$xGLjU3#IO2C zj5r7er_FgA&2I5L%$40Ea(lEuMU=jJBDx$4mLBr+gai*HYM)O9NgrhIc_|4y4-Q*Y zc#`M3TDa~*W4-=%B>@hC#CjwP?0t(+3>kndJ&Gi#uOnq?7KtHtK=Xv$uK@| zrqt#zWxfpeFo~v>c=8%0~$Lc^_(RQR&g_D^2|n$&3%4f z`Z2d&ZR)_>4?Hf^M=Te4xn**$LXH)^b*$Ss-M0BWo|&G?4de?8XC@BXfRa_vQ>QdF z5z37n>Pq12wy-3C8K3s+eZ)g2pBhp2_eff)+$b(p4nk=nuL>oMEeI8uzH{!YwZ6G# z)7Rjy(j+?Mg6`B&Zyu9>qZ6_3mpFfDS+*K}DE{W$0Y?vUM{yFR!A3wx2Oz`#(cnYC z16lINMm12hs^WoupqLBl6?GXt_R`x0ThNlE1Yv^~XNHz&KRvNa4ZU$48|P=Z-qW0k zxsn5K_H8-5y}|jt#o#}kQ%O5X_pzl)SFYXJB5krWzfbxeiO_clCG-vSyxxU~*&r`u0R^` zSEi1BG-rhnK)^*t0gncagf!>b`WJHpIFK3|CGhI|+(#1xIElb0XPAxRsS+h{_9_&{ z-Yb1!05glTdc^!euC@$;W&o7DQZro_#_fh}{9AuSuTK|)GWp7~J12MW@IYjNZJ9VI zNN`HvkRay+(hw*N9tZ!zKmZ5yd~`(zJ%sc-1+|#N!9m5erpZ8LOp5kRDc`-Nd-2z& z!;Y*x%EUl(RVAWgrwjmd?;s>LkROEuc!M-Qv%(A|uZKz)z!KlT*ggu95KuRKoe6k# z+N6GFqrUw!=99+k2<7w$&XvfEeePWz>Z~N}*Q-W9bM(u~tD2pRK9$KQt0)C{4xmukBEWn|kek?Q{bomM<28~YoO92~Swn3iq;Pchl21S;)!cNhK zlyeDCtyj1G$($(z@U;-1O`0TwNPO|0Zu69tH!69ANC~X|>+Mc^i7pfqrHlH9cYRRk zg~UTK!CxE6#i!f6N3OVjBAkbj$*4|$uv*O9wR8r0ro~rC2Yd}7V3TflYxQq}DwG{O zj4jgHzw`CqSzOFaV?hb}YoaT)ON}1VAV)WER**%G<$Dnf$QAX-DG(YY)@!suwej<{ z)rRi!UYTUdUuX*4MT~yy`pHBm39)OIlu`tw zhbLseZzN(<0h0Msa+{Ax++X}Dc@=Y{e7=4089X=iE_c?2&^IhY(C|RIMLG#MXV_3A zQUrfh==h7oAxy4UWiOV^}lJAW;WJEQPR2*!}T{So# zN3>v6Y5@wt&Gu6lvpkLip3D9r#a{$3Sv5AgpGZmAs75&CoPr)NZ9rw{((e_(iE8J+ z3_A8BD6xLEU-nEdUiM6a5>GBpI=xUF_;1&eRXS03@p4^$Df6%5kGMJO%OhapzknV4 z#OdL@C`J9%gnyGKaX)5*rJg!o*E*U7LYW-QllpyX+;@(=mKdJye@0niOpxUWXVDna z)F4}lxNKw3Grnx&?%uBZU~P9Xr|n)U*g^Rz?)2tWt`sqcG<7#{n^&oxC(@Fx^!wRT z&(YHEdumbZ)Mfi?a>Y`3WA3^AM$7$D_0s(TmD2q`<++1Bze2~E7VRhAdu&7GJ+=v7 zYbY-xcj4yuQ#gR`;Vgf}uRSj4Nx5?=zcm#{SzPTR>BwVkbHqUqyWmr&TGyXslxYUX z)aDpV+PED}^6es6+K*qvI$fJlSjlsPZJx8||)XBbq0!+!4?9p{s{q)nF`3pWd-5mx9CoX96K zP!?JfjH!mC88u{0ET49(oCb6b){GTBY&wae8r9j4^t`?(`gE_VkhPxVVd#q&J9ICr z0MrwQDZL_@(wLP~26~XuS4yXOGT0i>2L}>9YVABaZbLMeE}}Ynh~%Dsg;Cz}@{4|y zkL9mcuZGE3XW6-%Dp3h_So+v!NpL?AU|)bcL6{%S4lY1r2T!7uD)_LwCJx$+Ba>N8 zUGV-ETr8!NRwdaT9YzC=TN9LdZ#yq3rYh(Qud3pJI(baZmSjvuR1@j1ZGag`H;S!H zenL~Af~pVWtOaihQRObqlS{8c(!m1f`tAx~JVa^5Y>F9ks=mHOx1MYaUnps5pdw*u1JG4r=7kuhoh3+XJLbg@%lhEJ3BPFV6al`X6;P+HS znQ~*eM!T;SH}3?bKDt?zt+E89^oEZ8lIeiM(_npR62&{H)tkuWSezx3oyIgZ`64Q}YPUO`P z)t4I&>82n8O$>VyyO$o~*m1BToFlf>X<)TRU<8og!b8F~B<%a|Hbn!vJaTV*YD_h6 z#!)KkQQmo6j=!E0kWl>f7YZ~Y7v_p9c&Xa@Mx!g^=Bh+#bIKtvmPGbTxrgj4}>5MR7scqZGe$KD`{WYqr^5 zyO>9)=mjvnT&c@{>!pTx4tZZueq8oAQ6OC&!_xti%~r~H2{lt^jKh^j0eY@`Ks@K; z?P}e2_QtW4Kl#R~%M31MYjn)wcA8ADz3nceh$j3X=Z>P;`uv5d%0@s#ud{0c+X8a_ zBR9PU=V}Mfax>0z4>7+LRmFMc^7v00rUg~VzmyAN*Gyz$3GB`9Sb-_i6QWWMv810# zaG?61slwaOcRi#1VZXUHB~X1l2P`26c}D{c3s|)~R)-p=V4vWgQQq9X9kM)nJ|A5e zZTx&th1i`?GCUF~G*kP*=huU=mZt*vTm~zE{tnigj*1!H+pQ$rtx}kjx(f%?*7HVz zSf!4_i;gPJg+6Zj{fKd_F*fK<^S5q4Qll|COF{AntOh#I@-RSxrytX}-kE!y$;?nC zuvsLIbY^%~nZ)stl&!|##zhkQPOwo+c6~QMW37DJTGAjRRU{H^32)IHs?-v6B)6$yKLue zyU+t`viL`;YuUDOm|f0@C#jVpBk{P9Xr5X&_=i4Xn>rT7pavZcHCEil0w@LDgGpNs zN@NJ?G6VJ9R+m2jm&;@!tHlfjv6SzhsF0+LbqxjlvRblEc!A+=@cGd<{;yITY~zlx zM9f_hQ6Z0ddRXf4Knh-yZ)b;wrk-)VOrcl;Prp{M?@B!mg~9G84@+?;DL~m@Mla=}- zQz8n_(z7K4EA@P++DGnQ9?@5l?=2Ozf^XIR(4->+q(RY?kE2a(wOyI}V1cVHF2OU% zTd*Ny8q-gk-eD4T`r#ube=PRBlvc1LxPGeCX1zBwuB#z}y`9GzYvK zy{1Ucnj=(UZ(oDZo_NO_$ildRT6CbG-?YE{*Jh-HLH*@dv1IT1RQ->d+7FMdV~Ph5 z!iuG5eQf} zv7(nRzW_~I( zu>mn78J1OOdFGrUJ{t!i51fm!mYjygq&3kp%QXdW4-7KT0gj1>R&egkrhYgH%*qSk zZI8H5#joOF{}Rq!7zi4W@ETCOIQ{MLTIbjW7aw$_f?~dJcm04dqi%cjM&oSS?^M)l z(BjX!G;Wf*LaKIQy-9vhv8*f<9d|fZOMH6!KBO-ZR6J)Ln3g65RXxJJR1ji(}7=ijBdmWM`) z>-<(t)kUj_$wmF&f7z6}>betqe2JiJb^(vp(2pcJ%||utYZ}ykPhd*Y_)Z;EWqIcI z2o1{$c^?(sY(3nk`Ru7Do)4BAyR4f!nc;LR%1%SgJf#g(+@ehLeoEmlJ-gBcDCQJ> z&k(i|c;7sUaDNe3@LY5#hJ82jC@zR$&io60Xf|w6E1r2?uo1uMr^*d}yCZDxd<{;6 z+sKx>S=#y^gE0YWSAjU+_$(0b;7?bJQ*_PPJxXX3%kQ5wU77 zx1e^%+}i3Fwl{Bh`=*-25u>ZUsdSG-W~BXt&)Po zPkl6xV7pv@lL8aN*dH3JOEOmF!0~4LnGG2XUD3_Q5^BG1P5C>@N577|p;C`!xlI}C z7*X5(=N&~@GuACNFElXpY=FiG9C}mI*&Jw?%cUqz(t}Hr6l+Zn@uyVHuX8Q%UHRp9-#|;c9Sc+F>He zf7E_TP}YkZXXS&pht=`df(snN?X<6MdA}@w%D2WH&L(Eg)Pl-S>4D+(`DUk*P0;Me z7^k`x^#uy`mm>PVQcNEz(bJ=VE1#H?V)>N7LBW|+=g2oSf&^9g)K#?^-|cK?pUOx1 z3^Mm0(jHgnH-v)Z@C5uBv^cWce&637&Url&#+BRXAI9b8;DU6rjR%uaKiRoWo%*%O z$AnoICpTcmZ|KjnHN@J)@L8n*`i~3DXOTiFF$iGgYYge?=(MC0Bu4m3o#H@|Tn{zN zRfSi@h@_IR@VyB{18TC^fVZ%}Ky(_?UO$Rjc_>XYyynUcGi2XHrUIL1_NM}fO`7Gj zEItxT7Dp2`Ya*Ml>KE2e6FIi;<`yzWowxWcGg6)TH{NtLk1M%&(y*SLGef2H5$dTpNCTp{EVd}Ur{n(}7W_g)9A7c#@UE-RdWJjJ ztaKu$kiGk1s7#N!zZ?Ikm($q>aaVxqhUI>wPBu1-y*?VY*VPz{*__%4YOjKnmKx^Q z3tXxi);ELsht;M^DH_SCt8qbhpW(O(f>gLZnXi0-I!xBxUzomb_-;Y4K~+4ixP)A~ z83N8!noRl%TC1v9((s{DvKuj!tL4S96(x8~1+li_k!SB{6lIib7nK{dp5Nu8@;#0~ z^3vH%1Ek#auiIm^K96;_M9&j&fgsYSPYxBS&YCT8DHOlKV<$HsSbi&4zFN*^+n|2f zU_c815_mzQJDX$_Ha4GC_qlbxL3xr40^;j{{PelJ$yOVJZ3{|a)KzNH*YBcZo+l3P zzHlJ6E{JV^LoD_YeKEy(E9X_jZ}+bZ9(jA*P;OdGEIp?n0j|{amDKNTotDzGDoPvj zM}DHGCOf%(#IpXHbv#sez^~zkcru}4Lo`uGzQ$efZ_f|%d8}8H>6TJJ+AXt1yMfST zi;&MzS6+_};9S8QRt`A*rvQycwSzU;Qp`6&FmQIf~ zAIqF||5Xtn4n_8YfDiEKeJTz~WsQ<}1MIG!-)Oh1bkknNAIj4q?#-_>@f8z)QhCv# zI%1^`Kx5Zxt>iNRo164GH)-{Csek`dDj?!lR#%Rq05&M}=;87rx5TRQnG*#G_|r$5 zZ8`2hJXg@>5JGSu09gRUbx~UQw4a%}-(MK@E%^*knyv~)UJ!5A>gT(2k3|YZenKjg zZ?V-KL^H%Q5Z^R}u@Ok3dzY#9?)b3V@edmwz|wuWCW(N#S^=n;c&WfIiq(r*WVnN$F1LS5!n>a8$z!QYbudZUWLt zODwHB^o+VGf+I7m@Lm~QD0DORN%O~!080(SczTC)G$4U&&4@`4m70PzgEvP3_4cFf zG^VyBvCfa{%?Jd>p>rX@pU%tG3hFQ7&@BJ!t21pa7zT!e;BSim*Z?_eX5cn}DKaoH zFq+yJwhEBvT6Ft!>JWIc*LAzL{I;wA?RNZ{a2b!Jq*d@+nfgK=oGS6x*LzGPmqlf+ zWM%i;pI*5W27IIDkTLsU=qkh zyt9O^Na;1lUaFb*r);q@3|)3sB4z|CO833CqI$c8)y%h0wl)KABI09;uk}69=bbLT zTPURQRa@P&v-vA)lF4S|+(az?vq#+nqkKaH$ml! zYMbIX*-VMa^EaJTqc&UK!5?vrEQR73;_EI=7Nf4u@3h4x67*%rT8?olm@8zYtkr76 ztrodY1Ek}YZ#YC9L^$g#k*!yoD8zlUo_7j--js;H-2Kw|;VM3n+zv1?c>{u^_ckdx z3pj11ebs2|vlFF_ctI*m5Nr^CUu&mF7+dnmYU|iTMg5Qs-uBR%Xfg7^_1tgX@A>#v zgR>xI`x<1#s&H8v*TUlRX8|?(Ky0Dmjlav?G0FByM6PPc+-huu-_6&#T6vm><+RK8 zFK-tvhfEGQKUejrKk4isb4>-=F_#;Tv0W_^5h;&tN~c^De;x|Wcild4W76+vKW;44 zE}d&S{SNFCkYc@E@#7awWJVTEzLkf2@ukOl9DBqE1^hA5{^p)WB+)K)Sb{+JjkV|b zMVO#^A<+MI3v07N`$w)q7<%_-Jux~s5R5vE2C$u7nN;Xz$y>qYw5uQuB`rpAK_c0L z4WBI`m|B8lJG1pvis5Hg{|A%aqH^t0K8>8ZV*+!=D>h-z8RliyyxiXXHn7Mkm^$!RZ^o?UAta-cff0& zwfMy^K8;m3bZ0pBT<~Nl2;InEd6xA4=Uw#yAtFS70n7FK$f%ZPM$T9!9rvSQr$TWw zK|w*-g}&aNGPBcpC>&%5f)5L^>%b_2{#qbI+f)GFhN}v)$?`GKxwwI@T;5gX;k=Z& zPiibl5RZ;ak&A_kQ4(MErYi;w@OVJm4!;ddww$WKSoTA{nMm|GX^&W2Xvk9W2L)aD zoCYYxExGeo`$#0*%#`T*^`R|Y-Jh3iuLuAOM#i2*F>E+l20cAuP_zinx;&&lAwzUp ze0)xYz8%(dtnB5gyp2h%Ew*w=y(8xr=IAuiww}vNU9?h&k`Odm)gt9~i{DR=V|Z|j zp=W{jp)0oLN%3}yfdf@>fNQ2=VYuLNSe8Y`{R0sVed9w)gKZx@e-6Wyt353JK?4W0 zd&T&F^9|Ewcc||%Y9qjt*dD^T@wlY#;qR$?&V5}R-evE3NVR`{sH}^Z4s;$bIo^NT zHA0(3wHXKb-K^N`O7-Qa% zPYje+v+~YJ5O+2rV!P((Vn+8BY%xdOyx5on8TWiFF|mo-`PY6QE+-R~0RNNKS9&+f zy|_eW8<)?yKSM3Q#=`8i9xbFnQkaHT^uFPKQia2r54vdl!ESyD@d~;cz0ga)e^k`L zsqc&iX>rR(NqFt_ZS#1@2mb|z3h83*fkx$q%Stu8g}6ZDgG>-tjZude5k~GtcP*p8 z&i$pRB4YaN`SugLYA;dl?TjYPN)Q(d*rEHG&97JDWPaWE15wX?YWKsAPjFBDJT)A6 zbDdSlB=~DDnZB0i>T%#<8Y1K)X%@d5en5smJKv{TDE4oEhQh1*-(CrRb34@bPO1LA zsH~Qf&~Pd`6aQU}bgH4T+J-g*&~02cDHR(-qnHIuJ4{ey~VV)8ut7D6n3LGWy4 zr;2DVhWfhEL*mV`xTz20n`{oxXb6`0%Ya{hvnFqwbu9TpcHPpB1{5|N$?Vy@y;cFX zwf$p4--eHMlk7~izJDBL!~tOCa)iK+WK$n(f(Vr+jX6RR;Z&hV~ynj*KsgxbMF%ey@vu!zdTsTIC2Yi^RQo;+rM(wCao_>c2_G>1> zlpyiHgc(aOgL=!^2lEy0u|(4cTN{re;UNCAeIW~9TMlI`#p^Khd|om{nw;)?Dj^>O zFKT;UWOXoxD)Q-3vZP^(=>uNwi3k7&F)z2b>r3yt2On{Jdr_+Tty#aHb9?2;{pNGJ zx|6qFwo^?yd^*Nw`jVikqNpOyWE6FiS+`xO(}XF7E7`PS!O zZdBRcr|`M;UT!cC*2~Zzn^dc2ZlDvEh2p8coFz|u_WdE^y*slg5wQ)qc9+nl?K4=% z(P<)_ATAZhPt6U11#_NRnIQw?6hQrI$RCB=l66^a>ZnSM$J}<{kQ_}IU~n?XJFAWl znuT5g!MsF+dIaZJD-5{?c}wF$^j7c_;}^v-z~o3$1o*FO2I{cev=!ePxvhrYM=P#4 zTCN}U4{Z9C+s7$*d=!R-w0Zsj3xezXnQIx=T~u63=-QYLiv4Qy5s26Ve|NTLG^g8f zayIJow!K_pFzJ;Cbu;zko_4TfWP6$=Px`b>)I~L!o%;7&Z*25qSrl_hd8=jM$|#N9 z32zFi@)ata&$lI;MV_%}mlmHDhZ*@tX>m}B{N+a7R18yb{qUiX8D`i+#07R~8A3pz z;SXDe2|a1%Ia@-zp%iE1sn3IZL3&)26%sHm0x+O9;K0wAQa5N;YU-rDKrTk%D3FU@ zuC5_c!EVQbC!?J#tOEfZ5W3WX#A)T5VJOknD@aCtb2@c&6GI#X`P zeK7ToF``L5FE-P}lUer)bU;2UVkO0P6a-ItYAak%1M+;H)DKN@1>Q>NnnfK-kjVM1 zOs=_ax%;lI^_SQHm0-%ts&6H}tKZ)RQPai#A z-mD|&r2p0mvIjdoCiIyfEJ1?PXZH=0ibXe~Z~+#yh#Hav?YvDOY)W>H(8gRghboX!*Z zc$xS#nFoNK_a+vhHAX^9Y5bwnagj6?U|qhrFFY*C2Qv*D5KJI5i4>zW;3yhf3Uxkk zTIjb(p$CStjYjIuR3J`O?a#6sOR}QAe7y)gFA>n3DKrg~J)tv>>G}4aZOn-bpgK2n z-}CyM;NIgOIj*yAPTs+gj6nApw6fNhYt6ca6Oo%58;S-9xZ2;vA5Hp|=U0n}r6)4t zWMsb72oHg|s|8mwuRI5{i&CluC{M*BkpcQkIqYv}DVms;K4_j8VT6(bX!6j;E~5@x z&v)Ne*~mSGp{fFkEt1_`UxMAV1g>iRD(PoBT*%fmr*1nJ(OTqIf>E?kOMm!!TJ&*Y zNqx!KYjimnLx2NcU$;Gz^k$=6G~u=F7{eLk2b!Bx5%6AoA9VFw`9UigHYD_X+<^%; z^4=st!nb&^8A~y1809&tqJ=65c4ZxvWJ@;OIesmeI^2zv8LPF#FRV%$D{zp9n=cm+ z0~xpb-xJMTp9Xvo0#Uy2gaw9>z$S5(H%LJR*bC%(8GCw0m;m9!A1@}NLUf3XE^#qM zpd->v+WE^Bn3JaE^~h2#fT28wh_T9uO<#MVAaH=RV?N5IUU*hHZyu2hP}tL!JJNP7 zQ4$;E{wnU_R~C4ol2+T$7Hp8Uzt=(OuTem0^Soqinh2B)f#vD(P`bR#B1j-t^gHd&EFS;lG{+Vg}4prLeECOAp4|=rvIU zY`?rbzLx21BfS?#IQC`_Y|v_Lk$ISCWceWK+kN&{V6Y{v_S5V~#JP)~z;Vwap}_hoPgrmz&*w4R3}2cV0$LfN7Kbj?MO8vH1OQDC_RB%j9w6li zc}tnChB3dB|78i2Pc@vEEwP>V2=u9m10g*sRc8FCGi}%hlVVu=`e+GY*hZy4p6*$i z4hQkjNc;o8rhTS)!_Y{N!M}}$sht<=L>0sFCeOVkyfxKP z0ps}n8ueCR_B;Plr-@yL;>#r{1OqGsi{u+%wA>Usz2f7bdWr*g+N%me{N0Q>82eUf zmN{k0Jw@g|p59kH&;qMo&%5627R@q`#k!8spn!ul_U)LJF=&M~ue95s5bzB|(*-FK z^MSI?h*1SUU)XsayKXh7L6`e#$)BFmw4a$HEJld0qQ-Z$=-}1(^x`L`->Al;}^7Yd)_uOGEUaJYHO{g~Qt6 zT{`aJ#?PDDu@TQF$Yg+1OT3_uejo0RV|qe#e#V$Zg`P=ZY8baeUm?no*L%a#9eA3* zWwqNKVo~ZBPh$^zj{esj;mT2QES^>|w_wqcy-cN7z#z!8Hh5C9GV&>xRnyrth6fRA zTXVPaCCUW|&%v{kr6zaH^ObzN1VwxX3E)my|DaCLf|Lgnf=S8-uR3Q#vu|mh!(!05nEi*(tx~%vil*xMdp`-l%vFv$!ra*%t}ura zJ#Us*ZnMkw=O5Z+j>-#*T@hGFxq4{g0Fxm2ymkiG87GA!s5%K=#lula|<(v;*yh}ylCTLhV^Y!AnN`(bX!oPgR4`CaQk7Z&?6C^;#93^&;6z9InNjqaBhVR%&@mQ9fEgCK$>1nNH;ES*5}1s@BQ-oY{ZQS3 z%A$zkJa+S?nM}709z;tkWy-~mcuOeA^Cgf@6$+d}byl4ct~TQ~IvCuPgXUE4N&Yf} zlp)N0&>~`+NT)_c35eubULY)2sBM-vfL%A<<-4!M7sBvKU+75a8s+eq)-6 z1Dl?~uru5VuHptxp}Fa!J;FQaai-7d+i-42~ZDOd53*O!zn6r5nS50-oKSS@0!v^-)jzk^)`LoYdWM zPoTDo1jw4#BQ6RJul7zbSs*4M+g|#o4-&XH<6Bs<%=qQZ7WH!-E4R}FT(nX1R7+NO zVCwdVfFJ9-)C7l$^;~V+A#ug3=3R-* za%!5qb|eWr3vgfQ0`Zm>0%e5TpK}w5sSG6iv|6|+(T<|tR!OjWL$81ofD@E ziIIoCLM~hNdDDKkr349U@CAqw9j3wAkK;Jw1R+(-n$J$Wt2u$1;1S%rHZYoBt zP&XA9r!6%S@?=$r`sc!4?nSM6?^gT;yMiegQA-(HDTM_cys72tIn0#rf5(jQVs3tl zivvS9S&sN1EUW)9bwN6aD>Z-9kc!5>e&IVWvhSwxM;x5HpLA@o^(+U7szYd`d&TCt z!s%D6X32E?^jWDPABPyE5W)Fez&Rq~nie`X$L&^I_eyC7GtV0GrNljtf{MU~4EI;j z_yL-NXZq<;a{YbvHh`$M1$2+p>#fhqy5S{ zC3nNd1CV-mchqdmW4}?CiUZ7x6_PPTd1x_Nj5GToFfzBR@m*-LcpV_i__^n)x*4{D z`sruY0D4~R&DnAl(W0h+WX#;KM6Q>fD5D#hLLN)M=~glGIMk~jf*;3rv|!Ncu|WhM zH!*k^{%POS3~cwUJn-ii5@#JA6m%icHCJ}ccJ)G~49WAQ&DCIoMUo&ieHG1VO zhOnTf_Ao?eOk>z=XqTyrdvz55Vg+AbY=ykoQ9X=QCOvZy{zPxw*zO(X_b?i+3AxJM zG&l&`${PcMQCI*CjH`fIT2lV&Y`D%<%2Q~HnJ_Jb`a zp7Ye%mO)IcMMXE=8y$9uG)yWYuNu{vOUrPjQs$bbi$Pn^h5^VFe=0ZSaX0`>{g#Jp z`Van$QE~oZKg_1{v(H#=Oz)FJq{ zdE_F?)Sp^1{iO?6x)30HF`7c3s%wW>zl)lx)Y;b1mLt7K1yA7>)l=~KdryDNIAV=_ zD+N-(=M+3V;#gZwA#2J%MaVI6=Ojdf{-FZdLOYCGH5`2-z1(H*O#d{~ZT-Tt{C60N zEF%s-ejMw7QJ7;2`AlQTx~2YPV8dK}AEMEpSmLu3NX9cTn4J@$Nyf!&M4SpOQ7PNH1OszG%+zt^xjQVwY30lNhzFFB6 zPx5H$^DZjVH^PF~QPJLf*SlYvs-KY=GR0tMeV+I|gT{8Vt%hM)wC ze4VM3KTcl`%;v?v{!PKESFaTv{?jPHJdmpX_<1VP*5RUP8;>e4a@)wXDI(qg-18qWfJuqrqBn$(&P2f52J zE8b~jXH)CQkqR~SI-@Y;45+AVl^V*%CKT!99+!p%agPfKQ^ZSD}BGBj2A@*p1qE`c*0nLhPW4`Tai`x7QJ&Vl-B;E9Gc^eBw(h zhDAT-im@6zoTI65oC*4voG$nTa`ZY%nL~AK7PfG1YW3QmHZODZ{sVaW2*JgHTO{~_ zv*RwubFve(ovT(_sjvIIs^QMqFsNz7i$p1Jk(2tjw?D%1TxfebkmOt zFf&EgVwnsTa;lt7a9dm%70c5KFnlFhpVEfNEFf)R9n2=`)!&vILIcgm)MPmRZNfKi zCtLBr6EJH~4?_4tCM*o!h@Jr}O9Cm4SJ48g>|!#%@DxRx1|1AM2kXXIjY8b|?AkQ3 zVN*UNu8%;-v?`Lz`Y{mPeN z#QNxP`OdQC_Z)#-;cpJ_(ixq_DjhX@%{gxI+C}E{F3Q(XRl_^_RMI$mJCz(yznL$* zIPP=h`t}u%*x{f82<-5}vD>tx9bjHp3$ZoYM0; z-Z|Q>3_Kkl`zBAN0^$n9{e4sdxvJ=ANo9qRI6e|A6d7XlsWquPL2x&lL_hIo(1{B8 zUx?;_J~UAaDxje=4GiLrB8>drb(jj;`Fea)VHN7!}oF7cklb74dU+^ zg;6ay$a)A=!!?_%`^y4D*vFS{JjmyCbzsdKOoU(|pUK!OhJd0{4N0ct!Ylq->>2O= zIOiLXu|U=EcP~ulgxK%Oh@aLnAG3ZmGuBrDREMn1ph-z*W8dqgiTF>UE0dI_UYPcZ zUoac^TQC$2QjqMz_9RSxyr&m~A!mjrG!%p{cHi3pktA{&J&So*i$~-RIoE%m!|_y& z^auHpYr*NYL*0L^!Vkd=674E!fb#EfmGLn-%vxnI6_1%6D+9&*&C|}D_8!XF0MQrs5s#jBRd}AjM+HG`MQ+|iTBV=^y0yCX?=L-hw#ORI2smM zU3SouFCl3HqOy;AbRO|tss=tT@AXKNqp#I3zP6ie*qqMlDpA zP*@y@MYstD9in01t$UYCi&gD{Aqq~Q`rrH#Tq?ye;|qI58GL5HcuYpX;27d=yGjhQ zE&ml~BTEd^{WhWvZH89iRkud)ZQ>xSH3FmU-_`R%3yK`52o)98ynr9z5 zKyYkl%bmyt1i`rTW(u@J(~at!Qpe0gdtR`%tfihbSU zw;b@l_42;L@QYjT_OIl+a}A&TgGHJ+55}w}m)`|f21r|gLDZ(eu8`{Tf@y~rfD3%Odx zI?#aKh!Jf=mb6TjzoP5yf_qEfx2w^*J?Lgz=XlC#Hv}6}<2B)p!s^QK$tWI86>cijWIi^#i9cZMQ}=kHm<#O@Ae`vstPPyEaMP zvF=h>RlUOI!NEjGtUpWY+QkWjv;y#>xow-dRtD=m;oENQJPyUsoX*q7whLrNE~-QHs~3 z0pva7P{3IR6(tTITa_AU*|~Cm!UR!<(=i=j(gv}2Fdg#Zv*6(Ethu{Pd-lVUw((rJ z8<h z!Bp3AHGJJ|AYZZF=RJiO{>-7W2|}zje&i-JYG-~vIbBjIUL~LZ9QSn|wr0pOb&f6w z0`i|GQv;od!I4pWb!Ar`!z%c(H*{w@yoVR`SJKpO*y74wBo8W_Bk2rRf*Zh2uIpzB$6mTI;W@MH`c#TJweHozT^L)m7GH}aBTveM9%2BLeE&#{jJ8=F zTU+Y9dLaC8Nn6s=bVU-p7(}5u+T^1S+!u{Q7L(=Z*Hfn3e*?ZVgnUM%4Sn-^fAO6a*K+)8xx?=3>U! z7P__l!9g;7m#OyWK`%bbF{$IJoZ`+>=r{Hw^@!G+d>GQA{@A0{_L{|!1<5bPaCXl8x|31hnv+3rOqAu#~nH*?9*Py9pJj{@eGaKjXoXeP?BQl)mfNn z>aD34%$rxvh5e#&**;Ef%E+M}8i*YwWvd2vKJf21K&^%Cj2!M1FCd+#1$*_KGLORv z(Xl}5->_dx^gb7BdjGe$oVY{zXLrx3a-F&Mq@62=pKfA|Rr4sZb$^~?e?R#$>^DA) zzQOB_|ISt~dpL!#6Oy4puC1k>RC_)kLECG_+FLqQ?HVAcCZQfMJ>Ci_aogyXysbe@ zVUC*IE0w(cok!D>iA44!T0y_T_4ht8LQZ0BHLP(0Vy-6l96e@09T{O)f8omS~F`dYt~Vzh5^^U5hj)vs!WCQy(shf-W#Tuz^Otu}g1vYg@` zqWOe#+rwghOUxx8mIaX=U?_0e1|Avk=7z4!`~`KRL)mLh(j`a&^2rS(=$Q^EGF5u6 z_?`tXnp0{)&?!PaSxRwV6veG82J#+Fj6-22O%0%3;VE>AHejxbN0f<0R~M5b%k$1n z0;A2G_GWdm&T4$}Xg-;_igP=vKL<@_>UpOl-5UXudkrtlrkE{U%D7juq@` z(RK;fK22R&ZaZe$yGsne9nfZHw*2Fi%_v92g-BaHCWJ>8274J|DdfK*9cAY&_M`Y3XJO+6uYIym5(8~Trpp|^=6 ze^F7*!XF_&LA;t7;>>>2(G*KOWgXjZCHPG6Mnr>MPz{sK@CW#%3Ks$B`qjXv|7Z|N zH+-lipAfcchBKwdUX36eqQFbFx;^JLIbkfN`bKBga$O?;Ggx(1lX2iS!&XsgACiY{ zl>}=IvlyP{?XuUK6N~cI`Ci#+I+gd=UUE;lJOpHRmtuMk)IuNhO}KX)7)Nlx<9V)? z2!xy{h%tD~T=!Uell>DZbozq>3)?O8zh=U)9`5rr1nXa2{iXABTzjJ|t5BdY=i`+5 z{d-QViV8V#-N;xcR9`2~gID#}mFJG$_AL*-qcTaaW$`U~Clso)HfC!&%{5YCrfaj(pQ0+@e7C_Iw_Ah9tvOz}D8w7wc?q*h~-mB~e4O-meZFp5V| zW*q9zs+*5h$p@G;&zFGWz*4Wb0rWjfw5_S+?Ncqmi?eDUMq&o0h)mqu=jC1UwyKxH z2c(y*_1mKGII__T@E)uDPT(Xrq*s(Ij`1f!;l|7kUj?(_+|rT zw9JV1#LD={n)~4d6<|1YIB!tLYR0plzK&;Eg?4r*^R!}3EX^*MLB8el-Wqll_FhsTekh#7a>o_)1|9`U_?L)l|{W^Tn&auRG?hd zuqtIb+q`mk=vZ7}Zkt@*As|na1SrTC z+$UCP1HOOZ!Vsq}R0ON~WwVHfosm$g5;N%;3j| z)ggLodJ%+gy%kJV#<-sOn2&zDAk$|GsnTajbsT@M;um0VB*-2Is$UfP1G$Y+-%|p) z55DAkoHem__6$iaNT?oe^IzYFjz^#J7IAC|RK_h+3a-A(4Gs zNCj$s%!H84rU+OYFg(gLo@--Kz3@e?P{JzzpXdHFQ3NF0EjR|P{$c%r!C^v*=-&%9l^!(~3F*r#RYA$nZwQr>gw|kbLWJku zldkUOlbX!tC3__^7C?;um!W@a?Fyi6$F>rWH*-@q=mLIjay~3Ei;r<09+oou_nC0$ z%@YU$|6tJjh1M3o6 Date: Mon, 20 Mar 2023 23:32:06 +0800 Subject: [PATCH 02/28] [script] update warehouse jpa store metrics expire-time to 1h --- manager/src/main/resources/application-test.yml | 3 +++ .../docker-compose/hertzbeat-mysql-iotdb/conf/application.yml | 2 +- .../hertzbeat-mysql-tdengine/conf/application.yml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/manager/src/main/resources/application-test.yml b/manager/src/main/resources/application-test.yml index 446335826be..431306d5a98 100644 --- a/manager/src/main/resources/application-test.yml +++ b/manager/src/main/resources/application-test.yml @@ -41,6 +41,9 @@ spring: warehouse: store: + jpa: + enabled: true + expire-time: 1h td-engine: enabled: false driver-class-name: com.taosdata.jdbc.rs.RestfulDriver diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml index 79ca6b21f06..8165a2a9088 100644 --- a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml @@ -96,7 +96,7 @@ warehouse: # 存储历史数据方式, 下方只能enabled启用一种方式 jpa: enabled: false - expire-time: 7D + expire-time: 1h iot-db: enabled: true host: iotdb diff --git a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml index 14ef438806f..804d551e820 100644 --- a/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml +++ b/script/docker-compose/hertzbeat-mysql-tdengine/conf/application.yml @@ -96,7 +96,7 @@ warehouse: # 存储历史数据方式, 下方只能enabled启用一种方式 jpa: enabled: false - expire-time: 7D + expire-time: 1h td-engine: enabled: true driver-class-name: com.taosdata.jdbc.rs.RestfulDriver From 203a5324be01480e3b2559fa2e5e4fa3fad33891 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Mon, 20 Mar 2023 23:46:40 +0800 Subject: [PATCH 03/28] [script] update warehouse jpa store metrics expire-time to 1h --- .../current/start/iotdb-init.md | 2 +- .../current/start/tdengine-init.md | 2 +- .../usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/iotdb-init.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/iotdb-init.md index a4e3f79101a..ee503f95ade 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/iotdb-init.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/iotdb-init.md @@ -8,7 +8,7 @@ HertzBeat的历史数据存储依赖时序数据库 IoTDB 或 TDengine，任选 Apache IoTDB是一体化收集、存储、管理与分析物联网时序数据的软件系统，我们使用其存储分析采集到的监控指标历史数据。支持V0.12 - V0.13版本，推荐使用V0.13.*版本。 -**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能和高性能** +**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能，高性能和稳定性** > 如果您已有IoTDB环境，可直接跳到YML配置那一步。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/tdengine-init.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/tdengine-init.md index a3da8e1e96f..ff120e6247e 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/tdengine-init.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/tdengine-init.md @@ -8,7 +8,7 @@ HertzBeat的历史数据存储依赖时序数据库 TDengine 或 IoTDB，任选 TDengine是一款开源物联网时序型数据库，我们用其存储采集到的监控指标历史数据。 注意支持⚠️ 2.4.x版本。 -**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能和高性能** +**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能，高性能和稳定性** > 如果您已有TDengine环境，可直接跳到创建数据库实例那一步。 diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java index bfa39cf9ee5..ae3c192e5b6 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java @@ -64,8 +64,10 @@ public HistoryJpaDatabaseDataStorage(WarehouseProperties properties, this.historyDao = historyDao; } - @Scheduled( fixedDelay = 60, timeUnit = TimeUnit.MINUTES) + @Scheduled( fixedDelay = 10, timeUnit = TimeUnit.MINUTES) public void expiredDataCleaner() { + log.warn("[jpa-metrics-store]-start running expired data cleaner." + + "Please use time series db instead of jpa for better performance"); String expireTimeStr = jpaProperties.getExpireTime(); long expireTime = 0; try { From f7d53852e1c91bd02ce5488fe73a9977a64ac8a0 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Tue, 21 Mar 2023 00:20:22 +0800 Subject: [PATCH 04/28] [script] update warehouse jpa store metrics expire-time to 1h --- .../controller/MetricsDataController.java | 14 +++++++++++--- .../store/AbstractRealTimeDataStorage.java | 9 +++++++++ .../warehouse/store/DataStorageDispatch.java | 13 +++++++++++-- .../store/RealTimeMemoryDataStorage.java | 12 ++++++++++-- .../store/RealTimeRedisDataStorage.java | 16 +++++++++++----- 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java b/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java index 585cfc52252..2973f54fcb3 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java +++ b/warehouse/src/main/java/com/usthe/warehouse/controller/MetricsDataController.java @@ -27,6 +27,8 @@ import com.usthe.common.util.CommonConstants; import com.usthe.warehouse.store.AbstractHistoryDataStorage; import com.usthe.warehouse.store.AbstractRealTimeDataStorage; +import com.usthe.warehouse.store.HistoryJpaDatabaseDataStorage; +import com.usthe.warehouse.store.RealTimeMemoryDataStorage; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.tags.Tag; @@ -64,7 +66,14 @@ public class MetricsDataController { public MetricsDataController(List realTimeDataStorages, List historyDataStorages) { this.realTimeDataStorages = realTimeDataStorages; - this.historyDataStorages = historyDataStorages; + this.historyDataStorages = historyDataStorages.stream() + .filter(AbstractHistoryDataStorage::isServerAvailable).collect(Collectors.toList()); + if (this.historyDataStorages.size() > 1) { + this.historyDataStorages.removeIf(item -> item instanceof HistoryJpaDatabaseDataStorage); + } + if (this.realTimeDataStorages.size() > 1) { + this.realTimeDataStorages.removeIf(item -> item instanceof RealTimeMemoryDataStorage); + } } @GetMapping("/api/warehouse/storage/status") @@ -130,8 +139,7 @@ public ResponseEntity> getMetricHistoryData( @Parameter(description = "是否计算聚合数据,需查询时间段大于1周以上,默认不开启,聚合降样时间窗口默认为4小时", example = "false") @RequestParam(required = false) Boolean interval ) { - AbstractHistoryDataStorage historyDataStorage = historyDataStorages.stream() - .filter(AbstractHistoryDataStorage::isServerAvailable).findFirst().orElse(null); + AbstractHistoryDataStorage historyDataStorage = historyDataStorages.stream().findFirst().orElse(null); if (historyDataStorage == null) { return ResponseEntity.ok().body(new Message<>(FAIL_CODE, "time series database not available")); } diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/AbstractRealTimeDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/AbstractRealTimeDataStorage.java index 468743e751f..6c5f76853b8 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/AbstractRealTimeDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/AbstractRealTimeDataStorage.java @@ -30,6 +30,15 @@ @Slf4j public abstract class AbstractRealTimeDataStorage implements DisposableBean { + protected boolean serverAvailable; + + /** + * @return data storage是否可用 + */ + public boolean isServerAvailable() { + return serverAvailable; + } + /** * save collect metrics data * @param metricsData metrics data diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/DataStorageDispatch.java b/warehouse/src/main/java/com/usthe/warehouse/store/DataStorageDispatch.java index 6dfcf18ccc4..a8194aefcca 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/DataStorageDispatch.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/DataStorageDispatch.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Component; import java.util.List; +import java.util.stream.Collectors; /** * dispatch storage metrics data @@ -46,8 +47,10 @@ public DataStorageDispatch(CommonDataQueue commonDataQueue, List realTimeDataStorages) { this.commonDataQueue = commonDataQueue; this.workerPool = workerPool; - this.historyDataStorages = historyDataStorages; - this.realTimeDataStorages = realTimeDataStorages; + this.historyDataStorages = historyDataStorages.stream() + .filter(AbstractHistoryDataStorage::isServerAvailable).collect(Collectors.toList()); + this.realTimeDataStorages = realTimeDataStorages.stream() + .filter(AbstractRealTimeDataStorage::isServerAvailable).collect(Collectors.toList()); startStoragePersistentData(); startStorageRealTimeData(); } @@ -55,6 +58,9 @@ public DataStorageDispatch(CommonDataQueue commonDataQueue, private void startStorageRealTimeData() { Runnable runnable = () -> { Thread.currentThread().setName("warehouse-realtime-data-storage"); + if (realTimeDataStorages != null && realTimeDataStorages.size() > 1) { + realTimeDataStorages.removeIf(item -> item instanceof RealTimeMemoryDataStorage); + } while (!Thread.currentThread().isInterrupted()) { try { CollectRep.MetricsData metricsData = commonDataQueue.pollRealTimeStorageMetricsData(); @@ -74,6 +80,9 @@ private void startStorageRealTimeData() { protected void startStoragePersistentData() { Runnable runnable = () -> { Thread.currentThread().setName("warehouse-persistent-data-storage"); + if (historyDataStorages != null && historyDataStorages.size() > 1) { + historyDataStorages.removeIf(item -> item instanceof HistoryJpaDatabaseDataStorage); + } while (!Thread.currentThread().isInterrupted()) { try { CollectRep.MetricsData metricsData = commonDataQueue.pollPersistentStorageMetricsData(); diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeMemoryDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeMemoryDataStorage.java index a53710d74ae..995b87d2f9f 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeMemoryDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeMemoryDataStorage.java @@ -18,6 +18,7 @@ package com.usthe.warehouse.store; import com.usthe.common.entity.message.CollectRep; +import com.usthe.warehouse.config.WarehouseProperties; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.lang.NonNull; @@ -38,9 +39,16 @@ public class RealTimeMemoryDataStorage extends AbstractRealTimeDataStorage { private final Map metricsDataMap; + private static final Integer DEFAULT_INIT_SIZE = 1024; - public RealTimeMemoryDataStorage() { - metricsDataMap = new ConcurrentHashMap<>(1024); + public RealTimeMemoryDataStorage(WarehouseProperties properties) { + int initSize = DEFAULT_INIT_SIZE; + if (properties != null && properties.getStore() != null && properties.getStore().getMemory() != null + && properties.getStore().getMemory().getInitSize() != null) { + initSize = properties.getStore().getMemory().getInitSize(); + } + metricsDataMap = new ConcurrentHashMap<>(initSize); + this.serverAvailable = true; } @Override diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeRedisDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeRedisDataStorage.java index a2042eb7a9d..3a653f117b2 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeRedisDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/RealTimeRedisDataStorage.java @@ -49,7 +49,7 @@ public class RealTimeRedisDataStorage extends AbstractRealTimeDataStorage { private StatefulRedisConnection connection; public RealTimeRedisDataStorage(WarehouseProperties properties) { - initRedisClient(properties); + this.serverAvailable = initRedisClient(properties); } @Override @@ -79,10 +79,10 @@ public void saveData(CollectRep.MetricsData metricsData) { }); } - private void initRedisClient(WarehouseProperties properties) { + private boolean initRedisClient(WarehouseProperties properties) { if (properties == null || properties.getStore() == null || properties.getStore().getRedis() == null) { log.error("init error, please config Warehouse redis props in application.yml"); - throw new IllegalArgumentException("please config Warehouse redis props"); + return false; } WarehouseProperties.StoreProperties.RedisProperties redisProp = properties.getStore().getRedis(); RedisURI.Builder uriBuilder = RedisURI.builder() @@ -92,8 +92,14 @@ private void initRedisClient(WarehouseProperties properties) { if (redisProp.getPassword() != null && !"".equals(redisProp.getPassword())) { uriBuilder.withPassword(redisProp.getPassword().toCharArray()); } - redisClient = RedisClient.create(uriBuilder.build()); - connection = redisClient.connect(new MetricsDataRedisCodec()); + try { + redisClient = RedisClient.create(uriBuilder.build()); + connection = redisClient.connect(new MetricsDataRedisCodec()); + return true; + } catch (Exception e) { + log.error("init redis error {}", e.getMessage(), e); + } + return false; } @Override From 1adb54d2aa349594f85d0e48cc04b7205f5d43de Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Tue, 21 Mar 2023 00:30:21 +0800 Subject: [PATCH 05/28] [warehouse] support greptime db store metrics data --- warehouse/pom.xml | 7 + .../warehouse/config/WarehouseProperties.java | 66 ++++ .../store/HistoryGrepTimeDbDataStorage.java | 363 ++++++++++++++++++ 3 files changed, 436 insertions(+) create mode 100644 warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 6cfb6d5ab21..047bc1bb796 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -29,6 +29,7 @@ 3.4.0 3.0.5 3.0.0 + 0.1.0 4.0.0 @@ -79,6 +80,12 @@ iotdb-session ${iotdb-session.version} + + + io.greptime + greptimedb-all + ${greptimedb.version} + org.apache.kafka diff --git a/warehouse/src/main/java/com/usthe/warehouse/config/WarehouseProperties.java b/warehouse/src/main/java/com/usthe/warehouse/config/WarehouseProperties.java index 37582c9f7b1..f4111194934 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/config/WarehouseProperties.java +++ b/warehouse/src/main/java/com/usthe/warehouse/config/WarehouseProperties.java @@ -160,6 +160,10 @@ public static class StoreProperties { * IoTDB配置信息 */ private IotDbProperties iotDb; + /** + * GrepTimeDB Config + */ + private GreptimeProperties greptime; public JpaProperties getJpa() { return jpa; @@ -209,6 +213,14 @@ public void setIotDb(IotDbProperties iotDb) { this.iotDb = iotDb; } + public GreptimeProperties getGreptime() { + return greptime; + } + + public void setGreptime(GreptimeProperties greptime) { + this.greptime = greptime; + } + public static class MemoryProperties { /** * 内存数据存储是否启动 @@ -584,6 +596,60 @@ public void setExpireTime(String expireTime) { this.expireTime = expireTime; } } + + public static class GreptimeProperties { + /** + * Whether the GrepTimeDB data store is enabled + */ + private boolean enabled = false; + + /** + * GrepTimeDB endpoint + */ + private String endpoint = "127.0.0.1:4001"; + + /** + * GrepTimeDB username + */ + private String username; + + /** + * GrepTimeDB password + */ + private String password; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + } } } diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java new file mode 100644 index 00000000000..6e394cbbbc6 --- /dev/null +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.usthe.warehouse.store; + +import com.usthe.common.entity.dto.Value; +import com.usthe.common.entity.message.CollectRep; +import com.usthe.common.util.CommonConstants; +import com.usthe.warehouse.config.IotDbVersion; +import com.usthe.warehouse.config.WarehouseProperties; +import lombok.extern.slf4j.Slf4j; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.pool.SessionDataSetWrapper; +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.common.RowRecord; +import org.apache.iotdb.tsfile.write.record.Tablet; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.*; + +/** + * IoTDB data storage + * + * @author ceilzcx + * @since 2022/10/12 + */ +@Component +@ConditionalOnProperty(prefix = "warehouse.store.greptime", + name = "enabled", havingValue = "true") +@Slf4j +public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { + private static final String BACK_QUOTE = "`"; + private static final String DOUBLE_QUOTATION_MARKS = "\""; + /** + * set ttl never expire + */ + private static final String NEVER_EXPIRE = "-1"; + + /** + * storage group (存储组) + */ + private static final String STORAGE_GROUP = "root.hertzbeat"; + + private static final String SET_TTL = "set ttl to %s %s"; + + private static final String CANCEL_TTL = "unset ttl to %s"; + + private static final String SHOW_DEVICES = "SHOW DEVICES %s"; + + private static final String SHOW_STORAGE_GROUP = "show storage group"; + + private static final String QUERY_HISTORY_SQL + = "SELECT %s FROM %s WHERE Time >= now() - %s order by Time desc"; + private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL + = "SELECT FIRST_VALUE(%s), AVG(%s), MIN_VALUE(%s), MAX_VALUE(%s) FROM %s GROUP BY ([now() - %s, now()), 4h) WITHOUT NULL ANY"; + + private SessionPool sessionPool; + + private IotDbVersion version; + + private long queryTimeoutInMs; + + public HistoryGrepTimeDbDataStorage(WarehouseProperties properties) { + this.serverAvailable = this.initDbSession(properties.getStore().getGreptime()); + } + + private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProperties properties) { + String endpoint = "127.0.0.1:4001"; + return false; + } + + @Override + void saveData(CollectRep.MetricsData metricsData) { + if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) { + return; + } + if (metricsData.getValuesList().isEmpty()) { + log.info("[warehouse iotdb] flush metrics data {} is null, ignore.", metricsData.getId()); + return; + } + // tablet的deviceId添加引号会导致数据插入失败 + List schemaList = new ArrayList<>(); + + // todo MeasurementSchema是在客户端生成的数据结构，编码和压缩没有作用 + // todo 需要使用指定的数据结构，还是需要手动创建timeSeries或template + List fieldsList = metricsData.getFieldsList(); + for (CollectRep.Field field : fieldsList) { + MeasurementSchema schema = new MeasurementSchema(); + schema.setMeasurementId(field.getName()); + // handle field type + if (field.getType() == CommonConstants.TYPE_NUMBER) { + schema.setType(TSDataType.DOUBLE); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + schema.setType(TSDataType.TEXT); + } + schemaList.add(schema); + } + Map tabletMap = new HashMap<>(8); + try { + long now = System.currentTimeMillis(); + for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + String instance = valueRow.getInstance(); + if (!instance.isEmpty()) { + instance = String.format("\"%s\"", instance); + } + String deviceId = getDeviceId(metricsData.getApp(), metricsData.getMetrics(), metricsData.getId(), instance, false); + if (tabletMap.containsKey(instance)) { + // 避免Time重复 + now++; + } else { + tabletMap.put(instance, new Tablet(deviceId, schemaList)); + } + Tablet tablet = tabletMap.get(instance); + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, now); + for (int i = 0; i < fieldsList.size(); i++) { + if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { + if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) { + tablet.addValue(fieldsList.get(i).getName(), rowIndex, Double.parseDouble(valueRow.getColumns(i))); + } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) { + tablet.addValue(fieldsList.get(i).getName(), rowIndex, valueRow.getColumns(i)); + } + } else { + tablet.addValue(fieldsList.get(i).getName(), rowIndex, null); + } + } + } + for (Tablet tablet : tabletMap.values()) { + this.sessionPool.insertTablet(tablet, true); + } + } catch (StatementExecutionException | IoTDBConnectionException e) { + log.error(e.getMessage(), e); + } finally { + for (Tablet tablet : tabletMap.values()) { + tablet.reset(); + } + tabletMap.clear(); + } + } + + @Override + public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, + String instance, String history) { + Map> instanceValuesMap = new HashMap<>(8); + if (!isServerAvailable()) { + log.error("\n\t---------------IotDb Init Failed---------------\n" + + "\t--------------Please Config IotDb--------------\n" + + "\t----------Can Not Use Metric History Now----------\n"); + return instanceValuesMap; + } + String deviceId = getDeviceId(app, metrics, monitorId, instance, true); + String selectSql = ""; + try { + if (instance != null) { + selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId, history); + handleHistorySelect(selectSql, "", instanceValuesMap); + } else { + // 优先查询底下所有存在device, 如果存在底下所有device的数据, 否则查询deviceId的数据 + List devices = queryAllDevices(deviceId); + if (devices.isEmpty()) { + selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId, history); + handleHistorySelect(selectSql, "", instanceValuesMap); + } else { + // todo 改造成一个select查询: select device1.metric, device2.metric from xxx + for (String device : devices) { + String prefixDeviceId = getDeviceId(app, metrics, monitorId, null, false); + String instanceId = device.substring(prefixDeviceId.length() + 1); + selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId + "." + addQuote(instanceId), history); + handleHistorySelect(selectSql, instanceId, instanceValuesMap); + } + } + } + } catch (StatementExecutionException | IoTDBConnectionException e) { + log.error("select error history sql: {}", selectSql); + log.error(e.getMessage(), e); + } + return instanceValuesMap; + } + + private void handleHistorySelect(String selectSql, String instanceId, Map> instanceValuesMap) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSetWrapper dataSet = null; + try { + dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs); + log.debug("iot select sql: {}", selectSql); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long timestamp = rowRecord.getTimestamp(); + double value = rowRecord.getFields().get(0).getDoubleV(); + String strValue = BigDecimal.valueOf(value).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + List valueList = instanceValuesMap.computeIfAbsent(instanceId, k -> new LinkedList<>()); + valueList.add(new Value(strValue, timestamp)); + } + } finally { + if (dataSet != null) { + // 需要关闭结果集！！！否则会造成服务端堆积 + this.sessionPool.closeResultSet(dataSet); + } + } + } + + @Override + public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, + String metric, String instance, String history) { + Map> instanceValuesMap = new HashMap<>(8); + if (!isServerAvailable()) { + log.error("\n\t---------------IotDb Init Failed---------------\n" + + "\t--------------Please Config IotDb--------------\n" + + "\t----------Can Not Use Metric History Now----------\n"); + return instanceValuesMap; + } + String deviceId = getDeviceId(app, metrics, monitorId, instance, true); + String selectSql = ""; + if (instance != null) { + selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, + addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId, history); + handleHistoryIntervalSelect(selectSql, "", instanceValuesMap); + } else { + List devices = queryAllDevices(deviceId); + if (devices.isEmpty()) { + selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, + addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId, history); + handleHistoryIntervalSelect(selectSql, "", instanceValuesMap); + } else { + for (String device : devices) { + String prefixDeviceId = getDeviceId(app, metrics, monitorId, null, false); + String instanceId = device.substring(prefixDeviceId.length() + 1); + selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, + addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId + "." + addQuote(instanceId), history); + handleHistoryIntervalSelect(selectSql, instanceId, instanceValuesMap); + } + } + } + return instanceValuesMap; + } + + private void handleHistoryIntervalSelect(String selectSql, String instanceId, + Map> instanceValuesMap) { + SessionDataSetWrapper dataSet = null; + try { + dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs); + log.debug("iot select sql: {}", selectSql); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long timestamp = rowRecord.getTimestamp(); + double origin = rowRecord.getFields().get(0).getDoubleV(); + String originStr = BigDecimal.valueOf(origin).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + double avg = rowRecord.getFields().get(1).getDoubleV(); + String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + double min = rowRecord.getFields().get(2).getDoubleV(); + String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + double max = rowRecord.getFields().get(3).getDoubleV(); + String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + Value value = Value.builder() + .origin(originStr).mean(avgStr) + .min(minStr).max(maxStr) + .time(timestamp) + .build(); + List valueList = instanceValuesMap.computeIfAbsent(instanceId, k -> new LinkedList<>()); + valueList.add(value); + } + } catch (StatementExecutionException | IoTDBConnectionException e) { + log.error("select error history interval sql: {}", selectSql); + log.error(e.getMessage(), e); + } finally { + if (dataSet != null) { + // 需要关闭结果集！！！否则会造成服务端堆积 + this.sessionPool.closeResultSet(dataSet); + } + } + } + + /** + * 获取deviceId下的所有设备 + * + * @param deviceId 设备/实体 + */ + private List queryAllDevices(String deviceId) { + String showDevicesSql = String.format(SHOW_DEVICES, deviceId + ".*"); + SessionDataSetWrapper dataSet = null; + List devices = new ArrayList<>(); + try { + dataSet = this.sessionPool.executeQueryStatement(showDevicesSql, this.queryTimeoutInMs); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + devices.add(rowRecord.getFields().get(0).getStringValue()); + } + } catch (StatementExecutionException | IoTDBConnectionException e) { + log.error("query show all devices sql error. sql: {}", showDevicesSql); + log.error(e.getMessage(), e); + } finally { + if (dataSet != null) { + // 需要关闭结果集！！！否则会造成服务端堆积 + this.sessionPool.closeResultSet(dataSet); + } + } + return devices; + } + + /** + * 获取设备id + * 有instanceId的使用 ${group}.${app}.${metrics}.${monitor}.${instanceId} 的方式 + * 否则使用 ${group}.${app}.${metrics}.${monitor} 的方式 + * 查询时可以通过 ${group}.${app}.${metrics}.${monitor}.* 的方式获取所有instance数据 + */ + private String getDeviceId(String app, String metrics, Long monitorId, String instanceId, boolean useQuote) { + String deviceId = STORAGE_GROUP + "." + + (useQuote ? addQuote(app) : app) + "." + + (useQuote ? addQuote(metrics) : metrics) + "." + + ((IotDbVersion.V_1_0.equals(version) || useQuote) ? addQuote(monitorId.toString()) : monitorId.toString()); + if (instanceId != null && !instanceId.isEmpty() && !instanceId.equals(CommonConstants.NULL_VALUE)) { + deviceId += "." + addQuote(instanceId); + } + return deviceId; + } + + /** + * add quote，防止查询时关键字报错(eg: nodes) + */ + private String addQuote(String text) { + if (text == null || text.isEmpty() || (text.startsWith(BACK_QUOTE) && text.endsWith(BACK_QUOTE))) { + return text; + } + if ((text.startsWith(DOUBLE_QUOTATION_MARKS) && text.endsWith(DOUBLE_QUOTATION_MARKS))) { + if (IotDbVersion.V_1_0.equals(version)) { + text = text.replace("\"", "`"); + } + return text; + } + text = text.replace("'", "\\'"); + text = text.replace("\"", "\\\""); + text = text.replace("*", "-"); + text = String.format("`%s`", text); + return text; + } + + @Override + public void destroy() { + if (this.sessionPool != null) { + this.sessionPool.close(); + } + } +} From a1e7a50029a1ea73790f174cff0c16f090539c46 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Sat, 25 Mar 2023 14:35:47 +0800 Subject: [PATCH 06/28] [manager] update version --- pom.xml | 7 +++++++ warehouse/pom.xml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0ee31baad66..d3a7ec1d6e4 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ 1.7.21 1.6.11 2.7.4 + 3.4.0 2.3.0 @@ -52,6 +53,12 @@ springdoc-openapi-ui ${springdoc.version} + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 047bc1bb796..4a6146c60ea 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -29,7 +29,7 @@ 3.4.0 3.0.5 3.0.0 - 0.1.0 + 0.1.1 4.0.0 From 50050632f5dc4ba3f684d881fd728a6ab413c034 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Sun, 26 Mar 2023 15:12:41 +0800 Subject: [PATCH 07/28] [manager] update greptime --- .../store/HistoryGrepTimeDbDataStorage.java | 302 ++++----------- .../store/HistoryJpaDatabaseDataStorage.java | 358 +++++++++--------- 2 files changed, 250 insertions(+), 410 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index 6e394cbbbc6..98bab2a9547 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -20,23 +20,23 @@ import com.usthe.common.entity.dto.Value; import com.usthe.common.entity.message.CollectRep; import com.usthe.common.util.CommonConstants; -import com.usthe.warehouse.config.IotDbVersion; import com.usthe.warehouse.config.WarehouseProperties; import lombok.extern.slf4j.Slf4j; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; -import org.apache.iotdb.session.pool.SessionDataSetWrapper; -import org.apache.iotdb.session.pool.SessionPool; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.common.RowRecord; -import org.apache.iotdb.tsfile.write.record.Tablet; -import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import io.greptime.GreptimeDB; +import io.greptime.models.ColumnDataType; +import io.greptime.models.Err; +import io.greptime.models.Result; +import io.greptime.models.SemanticType; +import io.greptime.models.TableName; +import io.greptime.models.TableSchema; +import io.greptime.models.WriteOk; +import io.greptime.models.WriteRows; +import io.greptime.options.GreptimeOptions; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.*; +import java.util.concurrent.CompletableFuture; /** * IoTDB data storage @@ -49,17 +49,12 @@ name = "enabled", havingValue = "true") @Slf4j public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { - private static final String BACK_QUOTE = "`"; - private static final String DOUBLE_QUOTATION_MARKS = "\""; - /** - * set ttl never expire - */ - private static final String NEVER_EXPIRE = "-1"; /** - * storage group (存储组) + * storage database */ - private static final String STORAGE_GROUP = "root.hertzbeat"; + private static final String STORAGE_DATABASE = "hertzbeat"; + private static final String INSTANCE ="instance"; private static final String SET_TTL = "set ttl to %s %s"; @@ -74,19 +69,29 @@ public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = "SELECT FIRST_VALUE(%s), AVG(%s), MIN_VALUE(%s), MAX_VALUE(%s) FROM %s GROUP BY ([now() - %s, now()), 4h) WITHOUT NULL ANY"; - private SessionPool sessionPool; - - private IotDbVersion version; - - private long queryTimeoutInMs; + private GreptimeDB greptimeDb; public HistoryGrepTimeDbDataStorage(WarehouseProperties properties) { this.serverAvailable = this.initDbSession(properties.getStore().getGreptime()); } private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProperties properties) { - String endpoint = "127.0.0.1:4001"; - return false; + String endpoint = properties.getEndpoint(); + GreptimeOptions opts = GreptimeOptions.newBuilder(endpoint) + .writeMaxRetries(1) + .readMaxRetries(2) + .routeTableRefreshPeriodSeconds(-1) + .build(); + greptimeDb = new GreptimeDB(); + if (!greptimeDb.init(opts)) { + log.error("Fail to start GreptimeDB client"); + return false; + } + return createDatabase(); + } + private boolean createDatabase() { + // todo auto create database hertzbeat + return true; } @Override @@ -95,66 +100,70 @@ void saveData(CollectRep.MetricsData metricsData) { return; } if (metricsData.getValuesList().isEmpty()) { - log.info("[warehouse iotdb] flush metrics data {} is null, ignore.", metricsData.getId()); + log.info("[warehouse greptime] flush metrics data {} is null, ignore.", metricsData.getId()); return; } - // tablet的deviceId添加引号会导致数据插入失败 - List schemaList = new ArrayList<>(); - // todo MeasurementSchema是在客户端生成的数据结构，编码和压缩没有作用 - // todo 需要使用指定的数据结构，还是需要手动创建timeSeries或template + String monitorId = String.valueOf(metricsData.getId()); + String table = metricsData.getApp() + "_" + metricsData.getMetrics(); + TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); + + List semanticTypes = Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp); + List dataTypes = Arrays.asList(ColumnDataType.String, ColumnDataType.String, ColumnDataType.Int64); + List columnNames = Arrays.asList("monitor_id", "instance", "ts"); + List fieldsList = metricsData.getFieldsList(); for (CollectRep.Field field : fieldsList) { - MeasurementSchema schema = new MeasurementSchema(); - schema.setMeasurementId(field.getName()); + semanticTypes.add(SemanticType.Field); + columnNames.add(field.getName()); // handle field type if (field.getType() == CommonConstants.TYPE_NUMBER) { - schema.setType(TSDataType.DOUBLE); + dataTypes.add(ColumnDataType.Float64); } else if (field.getType() == CommonConstants.TYPE_STRING) { - schema.setType(TSDataType.TEXT); + dataTypes.add(ColumnDataType.String); } - schemaList.add(schema); } - Map tabletMap = new HashMap<>(8); + tableSchemaBuilder.semanticTypes(semanticTypes.toArray(new SemanticType[0])); + tableSchemaBuilder.dataTypes(dataTypes.toArray(new ColumnDataType[0])); + tableSchemaBuilder.columnNames(columnNames.toArray(new String[0])); + WriteRows rows = WriteRows.newBuilder(tableSchemaBuilder.build()).build(); try { long now = System.currentTimeMillis(); + Object[] values = new Object[3 + fieldsList.size()]; + values[0] = monitorId; + values[2] = now; for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { String instance = valueRow.getInstance(); if (!instance.isEmpty()) { instance = String.format("\"%s\"", instance); - } - String deviceId = getDeviceId(metricsData.getApp(), metricsData.getMetrics(), metricsData.getId(), instance, false); - if (tabletMap.containsKey(instance)) { - // 避免Time重复 - now++; + values[1] = instance; } else { - tabletMap.put(instance, new Tablet(deviceId, schemaList)); + values[1] = null; } - Tablet tablet = tabletMap.get(instance); - int rowIndex = tablet.rowSize++; - tablet.addTimestamp(rowIndex, now); for (int i = 0; i < fieldsList.size(); i++) { if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) { - tablet.addValue(fieldsList.get(i).getName(), rowIndex, Double.parseDouble(valueRow.getColumns(i))); + values[3 + i] = Double.parseDouble(valueRow.getColumns(i)); } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) { - tablet.addValue(fieldsList.get(i).getName(), rowIndex, valueRow.getColumns(i)); + values[3 + i] = valueRow.getColumns(i); } } else { - tablet.addValue(fieldsList.get(i).getName(), rowIndex, null); + values[3 + i] = null; } } + rows.insert(values); } - for (Tablet tablet : tabletMap.values()) { - this.sessionPool.insertTablet(tablet, true); - } - } catch (StatementExecutionException | IoTDBConnectionException e) { + rows.finish(); + CompletableFuture> writeFuture = greptimeDb.write(rows); + writeFuture.whenComplete((result, throwable) -> { + if (throwable != null) { + log.error("[warehouse greptime]-write data error:{}", result, throwable); + } else { + log.error("[warehouse greptime]-write data error:{}", result); + } + }); + } catch (Exception e) { log.error(e.getMessage(), e); - } finally { - for (Tablet tablet : tabletMap.values()) { - tablet.reset(); - } - tabletMap.clear(); } } @@ -163,201 +172,32 @@ public Map> getHistoryMetricData(Long monitorId, String app, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { - log.error("\n\t---------------IotDb Init Failed---------------\n" + + log.error("\n\t---------------GrepTime Init Failed---------------\n" + "\t--------------Please Config IotDb--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } - String deviceId = getDeviceId(app, metrics, monitorId, instance, true); - String selectSql = ""; - try { - if (instance != null) { - selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId, history); - handleHistorySelect(selectSql, "", instanceValuesMap); - } else { - // 优先查询底下所有存在device, 如果存在底下所有device的数据, 否则查询deviceId的数据 - List devices = queryAllDevices(deviceId); - if (devices.isEmpty()) { - selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId, history); - handleHistorySelect(selectSql, "", instanceValuesMap); - } else { - // todo 改造成一个select查询: select device1.metric, device2.metric from xxx - for (String device : devices) { - String prefixDeviceId = getDeviceId(app, metrics, monitorId, null, false); - String instanceId = device.substring(prefixDeviceId.length() + 1); - selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), deviceId + "." + addQuote(instanceId), history); - handleHistorySelect(selectSql, instanceId, instanceValuesMap); - } - } - } - } catch (StatementExecutionException | IoTDBConnectionException e) { - log.error("select error history sql: {}", selectSql); - log.error(e.getMessage(), e); - } return instanceValuesMap; } - private void handleHistorySelect(String selectSql, String instanceId, Map> instanceValuesMap) - throws IoTDBConnectionException, StatementExecutionException { - SessionDataSetWrapper dataSet = null; - try { - dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs); - log.debug("iot select sql: {}", selectSql); - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - long timestamp = rowRecord.getTimestamp(); - double value = rowRecord.getFields().get(0).getDoubleV(); - String strValue = BigDecimal.valueOf(value).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - List valueList = instanceValuesMap.computeIfAbsent(instanceId, k -> new LinkedList<>()); - valueList.add(new Value(strValue, timestamp)); - } - } finally { - if (dataSet != null) { - // 需要关闭结果集！！！否则会造成服务端堆积 - this.sessionPool.closeResultSet(dataSet); - } - } - } @Override public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { - log.error("\n\t---------------IotDb Init Failed---------------\n" + + log.error("\n\t---------------GrepTime Init Failed---------------\n" + "\t--------------Please Config IotDb--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } - String deviceId = getDeviceId(app, metrics, monitorId, instance, true); - String selectSql = ""; - if (instance != null) { - selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, - addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId, history); - handleHistoryIntervalSelect(selectSql, "", instanceValuesMap); - } else { - List devices = queryAllDevices(deviceId); - if (devices.isEmpty()) { - selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, - addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId, history); - handleHistoryIntervalSelect(selectSql, "", instanceValuesMap); - } else { - for (String device : devices) { - String prefixDeviceId = getDeviceId(app, metrics, monitorId, null, false); - String instanceId = device.substring(prefixDeviceId.length() + 1); - selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, - addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), deviceId + "." + addQuote(instanceId), history); - handleHistoryIntervalSelect(selectSql, instanceId, instanceValuesMap); - } - } - } return instanceValuesMap; } - private void handleHistoryIntervalSelect(String selectSql, String instanceId, - Map> instanceValuesMap) { - SessionDataSetWrapper dataSet = null; - try { - dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs); - log.debug("iot select sql: {}", selectSql); - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - long timestamp = rowRecord.getTimestamp(); - double origin = rowRecord.getFields().get(0).getDoubleV(); - String originStr = BigDecimal.valueOf(origin).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - double avg = rowRecord.getFields().get(1).getDoubleV(); - String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - double min = rowRecord.getFields().get(2).getDoubleV(); - String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - double max = rowRecord.getFields().get(3).getDoubleV(); - String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - Value value = Value.builder() - .origin(originStr).mean(avgStr) - .min(minStr).max(maxStr) - .time(timestamp) - .build(); - List valueList = instanceValuesMap.computeIfAbsent(instanceId, k -> new LinkedList<>()); - valueList.add(value); - } - } catch (StatementExecutionException | IoTDBConnectionException e) { - log.error("select error history interval sql: {}", selectSql); - log.error(e.getMessage(), e); - } finally { - if (dataSet != null) { - // 需要关闭结果集！！！否则会造成服务端堆积 - this.sessionPool.closeResultSet(dataSet); - } - } - } - - /** - * 获取deviceId下的所有设备 - * - * @param deviceId 设备/实体 - */ - private List queryAllDevices(String deviceId) { - String showDevicesSql = String.format(SHOW_DEVICES, deviceId + ".*"); - SessionDataSetWrapper dataSet = null; - List devices = new ArrayList<>(); - try { - dataSet = this.sessionPool.executeQueryStatement(showDevicesSql, this.queryTimeoutInMs); - while (dataSet.hasNext()) { - RowRecord rowRecord = dataSet.next(); - devices.add(rowRecord.getFields().get(0).getStringValue()); - } - } catch (StatementExecutionException | IoTDBConnectionException e) { - log.error("query show all devices sql error. sql: {}", showDevicesSql); - log.error(e.getMessage(), e); - } finally { - if (dataSet != null) { - // 需要关闭结果集！！！否则会造成服务端堆积 - this.sessionPool.closeResultSet(dataSet); - } - } - return devices; - } - - /** - * 获取设备id - * 有instanceId的使用 ${group}.${app}.${metrics}.${monitor}.${instanceId} 的方式 - * 否则使用 ${group}.${app}.${metrics}.${monitor} 的方式 - * 查询时可以通过 ${group}.${app}.${metrics}.${monitor}.* 的方式获取所有instance数据 - */ - private String getDeviceId(String app, String metrics, Long monitorId, String instanceId, boolean useQuote) { - String deviceId = STORAGE_GROUP + "." + - (useQuote ? addQuote(app) : app) + "." + - (useQuote ? addQuote(metrics) : metrics) + "." + - ((IotDbVersion.V_1_0.equals(version) || useQuote) ? addQuote(monitorId.toString()) : monitorId.toString()); - if (instanceId != null && !instanceId.isEmpty() && !instanceId.equals(CommonConstants.NULL_VALUE)) { - deviceId += "." + addQuote(instanceId); - } - return deviceId; - } - - /** - * add quote，防止查询时关键字报错(eg: nodes) - */ - private String addQuote(String text) { - if (text == null || text.isEmpty() || (text.startsWith(BACK_QUOTE) && text.endsWith(BACK_QUOTE))) { - return text; - } - if ((text.startsWith(DOUBLE_QUOTATION_MARKS) && text.endsWith(DOUBLE_QUOTATION_MARKS))) { - if (IotDbVersion.V_1_0.equals(version)) { - text = text.replace("\"", "`"); - } - return text; - } - text = text.replace("'", "\\'"); - text = text.replace("\"", "\\\""); - text = text.replace("*", "-"); - text = String.format("`%s`", text); - return text; - } - @Override public void destroy() { - if (this.sessionPool != null) { - this.sessionPool.close(); + if (this.greptimeDb != null) { + this.greptimeDb.shutdownGracefully(); } } } diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java index 1ac0bf19bc5..a48a571e9f3 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java @@ -52,193 +52,193 @@ name = "enabled", havingValue = "true") @Slf4j public class HistoryJpaDatabaseDataStorage extends AbstractHistoryDataStorage { - private HistoryDao historyDao; - private WarehouseProperties.StoreProperties.JpaProperties jpaProperties; + private HistoryDao historyDao; + private WarehouseProperties.StoreProperties.JpaProperties jpaProperties; - private static final int STRING_MAX_LENGTH = 1024; - private static final int MAX_HISTORY_TABLE_RECORD = 60_000; + private static final int STRING_MAX_LENGTH = 1024; + private static final int MAX_HISTORY_TABLE_RECORD = 60_000; - public HistoryJpaDatabaseDataStorage(WarehouseProperties properties, - HistoryDao historyDao) { - this.jpaProperties = properties.getStore().getJpa(); - this.serverAvailable = true; - this.historyDao = historyDao; - } + public HistoryJpaDatabaseDataStorage(WarehouseProperties properties, + HistoryDao historyDao) { + this.jpaProperties = properties.getStore().getJpa(); + this.serverAvailable = true; + this.historyDao = historyDao; + } - @Scheduled( fixedDelay = 1, timeUnit = TimeUnit.MINUTES) - public void expiredDataCleaner() { - log.warn("[jpa-metrics-store]-start running expired data cleaner." + - "Please use time series db instead of jpa for better performance"); - String expireTimeStr = jpaProperties.getExpireTime(); - long expireTime = 0; - try { - if (NumberUtils.isParsable(expireTimeStr)) { - expireTime = NumberUtils.toLong(expireTimeStr); - expireTime = (ZonedDateTime.now().toEpochSecond() + expireTime) * 1000; - } else { - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(expireTimeStr); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - expireTime = dateTime.toEpochSecond() * 1000; - } - } catch (Exception e) { - log.error("expiredDataCleaner time error: {}. use default expire time to clean: 1h", e.getMessage()); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(1)); - expireTime = dateTime.toEpochSecond() * 1000; - } - try { - int rows = historyDao.deleteHistoriesByTimeBefore(expireTime); - log.info("[jpa-metrics-store]-delete {} rows.", rows); - long total = historyDao.count(); - if (total > MAX_HISTORY_TABLE_RECORD) { - rows = historyDao.deleteOlderHistoriesRecord(); - log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows); - } - } catch (Exception e) { - log.error("expiredDataCleaner database error: {}.", e.getMessage()); - log.error("try to truncate table hzb_history. Please use time series db instead of jpa for better performance."); - historyDao.truncateTable(); - } - } + @Scheduled( fixedDelay = 1, timeUnit = TimeUnit.MINUTES) + public void expiredDataCleaner() { + log.warn("[jpa-metrics-store]-start running expired data cleaner." + + "Please use time series db instead of jpa for better performance"); + String expireTimeStr = jpaProperties.getExpireTime(); + long expireTime = 0; + try { + if (NumberUtils.isParsable(expireTimeStr)) { + expireTime = NumberUtils.toLong(expireTimeStr); + expireTime = (ZonedDateTime.now().toEpochSecond() + expireTime) * 1000; + } else { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(expireTimeStr); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + expireTime = dateTime.toEpochSecond() * 1000; + } + } catch (Exception e) { + log.error("expiredDataCleaner time error: {}. use default expire time to clean: 1h", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(1)); + expireTime = dateTime.toEpochSecond() * 1000; + } + try { + int rows = historyDao.deleteHistoriesByTimeBefore(expireTime); + log.info("[jpa-metrics-store]-delete {} rows.", rows); + long total = historyDao.count(); + if (total > MAX_HISTORY_TABLE_RECORD) { + rows = historyDao.deleteOlderHistoriesRecord(); + log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows); + } + } catch (Exception e) { + log.error("expiredDataCleaner database error: {}.", e.getMessage()); + log.error("try to truncate table hzb_history. Please use time series db instead of jpa for better performance."); + historyDao.truncateTable(); + } + } - @Override - void saveData(CollectRep.MetricsData metricsData) { - if (metricsData.getCode() != CollectRep.Code.SUCCESS) { - return; - } - if (metricsData.getValuesList().isEmpty()) { - log.info("[warehouse jpa] flush metrics data {} is null, ignore.", metricsData.getId()); - return; - } - String monitorType = metricsData.getApp(); - String metrics = metricsData.getMetrics(); - List fieldsList = metricsData.getFieldsList(); - try { - List historyList = new LinkedList<>(); - History.HistoryBuilder historyBuilder = History.builder() - .monitorId(metricsData.getId()) - .app(monitorType) - .metrics(metrics) - .time(metricsData.getTime()); - for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { - String instance = valueRow.getInstance(); - if (!instance.isEmpty()) { - instance = formatStrValue(instance); - historyBuilder.instance(instance); - } else { - historyBuilder.instance(null); - } - for (int i = 0; i < fieldsList.size(); i++) { - CollectRep.Field field = fieldsList.get(i); - // ignore string value store in db - if (field.getType() == CommonConstants.TYPE_STRING) { - continue; - } - historyBuilder.metric(field.getName()); - if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { - if (field.getType() == CommonConstants.TYPE_NUMBER) { - historyBuilder.metricType(CommonConstants.TYPE_NUMBER) - .dou(Double.parseDouble(valueRow.getColumns(i))); - } else if (field.getType() == CommonConstants.TYPE_STRING) { - historyBuilder.metricType(CommonConstants.TYPE_STRING) - .str(formatStrValue(valueRow.getColumns(i))); - } - } else { - if (field.getType() == CommonConstants.TYPE_NUMBER) { - historyBuilder.metricType(CommonConstants.TYPE_NUMBER).dou(null); - } else if (field.getType() == CommonConstants.TYPE_STRING) { - historyBuilder.metricType(CommonConstants.TYPE_STRING).str(null); - } - } - historyList.add(historyBuilder.build()); - } - } - historyDao.saveAll(historyList); - } catch (Exception e) { - log.error(e.getMessage(), e); - } + @Override + void saveData(CollectRep.MetricsData metricsData) { + if (metricsData.getCode() != CollectRep.Code.SUCCESS) { + return; + } + if (metricsData.getValuesList().isEmpty()) { + log.info("[warehouse jpa] flush metrics data {} is null, ignore.", metricsData.getId()); + return; + } + String monitorType = metricsData.getApp(); + String metrics = metricsData.getMetrics(); + List fieldsList = metricsData.getFieldsList(); + try { + List historyList = new LinkedList<>(); + History.HistoryBuilder historyBuilder = History.builder() + .monitorId(metricsData.getId()) + .app(monitorType) + .metrics(metrics) + .time(metricsData.getTime()); + for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + String instance = valueRow.getInstance(); + if (!instance.isEmpty()) { + instance = formatStrValue(instance); + historyBuilder.instance(instance); + } else { + historyBuilder.instance(null); } - - /** - * 从数据库获取指标历史数据 - * - * @param monitorId 监控ID - * @param app 监控类型 - * @param metrics 指标集合名 - * @param metric 指标名 - * @param instance 实例 - * @param history 历史范围 - * @return 指标历史数据列表 - */ - @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { - Map> instanceValuesMap = new HashMap<>(8); - Specification specification = (root, query, criteriaBuilder) -> { - List andList = new ArrayList<>(); - Predicate predicateMonitorId = criteriaBuilder.equal(root.get("monitorId"), monitorId); - Predicate predicateMonitorType = criteriaBuilder.equal(root.get("app"), app); - Predicate predicateMonitorMetrics = criteriaBuilder.equal(root.get("metrics"), metrics); - Predicate predicateMonitorMetric = criteriaBuilder.equal(root.get("metric"), metric); - andList.add(predicateMonitorId); - andList.add(predicateMonitorType); - andList.add(predicateMonitorMetrics); - andList.add(predicateMonitorMetric); - if (instance != null && !"".equals(instance)) { - Predicate predicateMonitorInstance = criteriaBuilder.equal(root.get("instance"), instance); - andList.add(predicateMonitorInstance); - } - if (history != null) { - try { - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - long timeBefore = dateTime.toEpochSecond() * 1000; - Predicate timePredicate = criteriaBuilder.ge(root.get("time"), timeBefore); - andList.add(timePredicate); - } catch (Exception e) { - log.error(e.getMessage()); - } - } - Predicate[] predicates = new Predicate[andList.size()]; - Predicate predicate = criteriaBuilder.and(andList.toArray(predicates)); - return query.where(predicate).getRestriction(); - }; - Sort sortExp = Sort.by(new Sort.Order(Sort.Direction.DESC, "time")); - List historyList = historyDao.findAll(specification, sortExp); - for (History dataItem : historyList) { - String value = ""; - if (dataItem.getMetricType() == CommonConstants.TYPE_NUMBER) { - if (dataItem.getDou() != null) { - value = BigDecimal.valueOf(dataItem.getDou()).setScale(4, RoundingMode.HALF_UP) - .stripTrailingZeros().toPlainString(); - } - } else { - value = dataItem.getStr(); - } - String instanceValue = dataItem.getInstance() == null ? "" : dataItem.getInstance(); - List valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); - valueList.add(new Value(value, dataItem.getTime())); - } - return instanceValuesMap; + for (int i = 0; i < fieldsList.size(); i++) { + CollectRep.Field field = fieldsList.get(i); + // ignore string value store in db + if (field.getType() == CommonConstants.TYPE_STRING) { + continue; + } + historyBuilder.metric(field.getName()); + if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { + if (field.getType() == CommonConstants.TYPE_NUMBER) { + historyBuilder.metricType(CommonConstants.TYPE_NUMBER) + .dou(Double.parseDouble(valueRow.getColumns(i))); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + historyBuilder.metricType(CommonConstants.TYPE_STRING) + .str(formatStrValue(valueRow.getColumns(i))); + } + } else { + if (field.getType() == CommonConstants.TYPE_NUMBER) { + historyBuilder.metricType(CommonConstants.TYPE_NUMBER).dou(null); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + historyBuilder.metricType(CommonConstants.TYPE_STRING).str(null); + } + } + historyList.add(historyBuilder.build()); } + } + historyDao.saveAll(historyList); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } - private String formatStrValue(String value) { - if (value == null) { - return ""; - } - value = value.replace("'", "\\'"); - value = value.replace("\"", "\\\""); - value = value.replace("*", "-"); - value = String.format("`%s`", value); - if (value.length() > STRING_MAX_LENGTH) { - value = value.substring(0, STRING_MAX_LENGTH - 1); - } - return value; + /** + * 从数据库获取指标历史数据 + * + * @param monitorId 监控ID + * @param app 监控类型 + * @param metrics 指标集合名 + * @param metric 指标名 + * @param instance 实例 + * @param history 历史范围 + * @return 指标历史数据列表 + */ + @Override + public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { + Map> instanceValuesMap = new HashMap<>(8); + Specification specification = (root, query, criteriaBuilder) -> { + List andList = new ArrayList<>(); + Predicate predicateMonitorId = criteriaBuilder.equal(root.get("monitorId"), monitorId); + Predicate predicateMonitorType = criteriaBuilder.equal(root.get("app"), app); + Predicate predicateMonitorMetrics = criteriaBuilder.equal(root.get("metrics"), metrics); + Predicate predicateMonitorMetric = criteriaBuilder.equal(root.get("metric"), metric); + andList.add(predicateMonitorId); + andList.add(predicateMonitorType); + andList.add(predicateMonitorMetrics); + andList.add(predicateMonitorMetric); + if (instance != null && !"".equals(instance)) { + Predicate predicateMonitorInstance = criteriaBuilder.equal(root.get("instance"), instance); + andList.add(predicateMonitorInstance); + } + if (history != null) { + try { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + long timeBefore = dateTime.toEpochSecond() * 1000; + Predicate timePredicate = criteriaBuilder.ge(root.get("time"), timeBefore); + andList.add(timePredicate); + } catch (Exception e) { + log.error(e.getMessage()); } - - @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { - return new HashMap<>(8); + } + Predicate[] predicates = new Predicate[andList.size()]; + Predicate predicate = criteriaBuilder.and(andList.toArray(predicates)); + return query.where(predicate).getRestriction(); + }; + Sort sortExp = Sort.by(new Sort.Order(Sort.Direction.DESC, "time")); + List historyList = historyDao.findAll(specification, sortExp); + for (History dataItem : historyList) { + String value = ""; + if (dataItem.getMetricType() == CommonConstants.TYPE_NUMBER) { + if (dataItem.getDou() != null) { + value = BigDecimal.valueOf(dataItem.getDou()).setScale(4, RoundingMode.HALF_UP) + .stripTrailingZeros().toPlainString(); } + } else { + value = dataItem.getStr(); + } + String instanceValue = dataItem.getInstance() == null ? "" : dataItem.getInstance(); + List valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); + valueList.add(new Value(value, dataItem.getTime())); + } + return instanceValuesMap; + } + + private String formatStrValue(String value) { + if (value == null) { + return ""; + } + value = value.replace("'", "\\'"); + value = value.replace("\"", "\\\""); + value = value.replace("*", "-"); + value = String.format("`%s`", value); + if (value.length() > STRING_MAX_LENGTH) { + value = value.substring(0, STRING_MAX_LENGTH - 1); + } + return value; + } + + @Override + public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { + return new HashMap<>(8); + } - @Override - public void destroy() throws Exception {} + @Override + public void destroy() throws Exception {} } From c2ef9a84570a9bcc92c8141820c92c158d09de96 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Sun, 26 Mar 2023 16:14:33 +0800 Subject: [PATCH 08/28] [manager] update greptime --- warehouse/pom.xml | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 4a6146c60ea..1e8a8a388a9 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -83,8 +83,49 @@ io.greptime - greptimedb-all + greptimedb-protocol ${greptimedb.version} + + + io.grpc + grpc-all + + + com.google.code.gson + gson + + + com.google.guava + guava + + + com.google.protobuf + protobuf-java + + + + + io.greptime + greptimedb-grpc + ${greptimedb.version} + + + io.grpc + grpc-all + + + com.google.code.gson + gson + + + com.google.guava + guava + + + com.google.protobuf + protobuf-java + + From 9c2032899d1be41a19bec184d2f4131cb3c04bdc Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Sun, 26 Mar 2023 17:42:48 +0800 Subject: [PATCH 09/28] constraint common dependencies version --- collector/pom.xml | 5 +---- common/pom.xml | 3 --- manager/pom.xml | 2 -- pom.xml | 32 ++++++++++++++++++++++++++++++++ warehouse/pom.xml | 2 -- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/collector/pom.xml b/collector/pom.xml index 2101a9ee48d..418314ccf5c 100644 --- a/collector/pom.xml +++ b/collector/pom.xml @@ -47,7 +47,6 @@ com.usthe.tancloud common 1.0 - @@ -93,12 +92,10 @@ com.google.guava guava - 31.0.1-jre com.google.code.gson gson - 2.8.9 @@ -162,7 +159,7 @@ org.mongodb mongodb-driver-sync - 4.8.1 + 4.6.1 diff --git a/common/pom.xml b/common/pom.xml index b3260f05294..7c8eb041167 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -94,17 +94,14 @@ com.google.guava guava - ${guava.version} com.google.code.gson gson - ${gson.version} com.google.protobuf protobuf-java-util - ${protobuf-java-util.version} com.google.guava diff --git a/manager/pom.xml b/manager/pom.xml index 9cacab9827e..b684e8575fb 100644 --- a/manager/pom.xml +++ b/manager/pom.xml @@ -36,7 +36,6 @@ 3.3.0 8.0.28 1.9.22 - 1.32 1.0.8 1.0 @@ -120,7 +119,6 @@ org.yaml snakeyaml - ${snake.yaml.version} diff --git a/pom.xml b/pom.xml index 0ee31baad66..bede1d1fdb1 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,11 @@ 1.6.11 2.7.4 2.3.0 + 1.32 + 3.4.0 + 2.8.9 + 31.0.1-jre + 3.19.1 @@ -52,6 +57,33 @@ springdoc-openapi-ui ${springdoc.version} + + org.yaml + snakeyaml + ${snake.yaml.version} + + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + + + + com.google.guava + guava + ${guava.version} + + + com.google.code.gson + gson + ${gson.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf-java-util.version} + diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 6cfb6d5ab21..13f857f98ef 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -26,7 +26,6 @@ 1.0 0.13.3 - 3.4.0 3.0.5 3.0.0 @@ -83,7 +82,6 @@ org.apache.kafka kafka-clients - ${kafka-clients.version} From cf32819c49208ac02860747a5618f011879dec04 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Mon, 27 Mar 2023 23:49:17 +0800 Subject: [PATCH 10/28] fix unsupported error --- .../usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index 98bab2a9547..62fc502c7ed 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -108,9 +108,9 @@ void saveData(CollectRep.MetricsData metricsData) { String table = metricsData.getApp() + "_" + metricsData.getMetrics(); TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); - List semanticTypes = Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp); - List dataTypes = Arrays.asList(ColumnDataType.String, ColumnDataType.String, ColumnDataType.Int64); - List columnNames = Arrays.asList("monitor_id", "instance", "ts"); + List semanticTypes = new LinkedList<>(Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp)); + List dataTypes = new LinkedList<>(Arrays.asList(ColumnDataType.String, ColumnDataType.String, ColumnDataType.Int64)); + List columnNames = new LinkedList<>(Arrays.asList("monitor_id", "instance", "ts")); List fieldsList = metricsData.getFieldsList(); for (CollectRep.Field field : fieldsList) { From 1b20bf372157ba017efd7134d6250413b3387c82 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Mon, 27 Mar 2023 23:58:02 +0800 Subject: [PATCH 11/28] remove success result --- .../com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index 62fc502c7ed..221c03df646 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -158,8 +158,6 @@ void saveData(CollectRep.MetricsData metricsData) { writeFuture.whenComplete((result, throwable) -> { if (throwable != null) { log.error("[warehouse greptime]-write data error:{}", result, throwable); - } else { - log.error("[warehouse greptime]-write data error:{}", result); } }); } catch (Exception e) { From 901c98ccf5b18323227df5813de1fed2f74381c4 Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Wed, 29 Mar 2023 21:54:53 +0800 Subject: [PATCH 12/28] [code]feature: add GrepTimeDb `getHistoryMetricData` --- warehouse/pom.xml | 7 ++- .../store/HistoryGrepTimeDbDataStorage.java | 63 +++++++++++++++---- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/warehouse/pom.xml b/warehouse/pom.xml index f7adf0ee262..e8cd44839b2 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -28,7 +28,7 @@ 0.13.3 3.0.5 3.0.0 - 0.1.1 + 0.1.2 4.0.0 @@ -80,6 +80,11 @@ ${iotdb-session.version} + + + + + io.greptime greptimedb-protocol diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index 221c03df646..e73f32c3c72 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -21,22 +21,20 @@ import com.usthe.common.entity.message.CollectRep; import com.usthe.common.util.CommonConstants; import com.usthe.warehouse.config.WarehouseProperties; +import io.greptime.models.*; import lombok.extern.slf4j.Slf4j; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import io.greptime.GreptimeDB; -import io.greptime.models.ColumnDataType; -import io.greptime.models.Err; -import io.greptime.models.Result; -import io.greptime.models.SemanticType; -import io.greptime.models.TableName; -import io.greptime.models.TableSchema; -import io.greptime.models.WriteOk; -import io.greptime.models.WriteRows; import io.greptime.options.GreptimeOptions; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.*; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; /** * IoTDB data storage @@ -65,7 +63,7 @@ public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { private static final String SHOW_STORAGE_GROUP = "show storage group"; private static final String QUERY_HISTORY_SQL - = "SELECT %s FROM %s WHERE Time >= now() - %s order by Time desc"; + = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= now() - INTERVAL %s order by ts desc;"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = "SELECT FIRST_VALUE(%s), AVG(%s), MIN_VALUE(%s), MAX_VALUE(%s) FROM %s GROUP BY ([now() - %s, now()), 4h) WITHOUT NULL ANY"; @@ -105,7 +103,8 @@ void saveData(CollectRep.MetricsData metricsData) { } String monitorId = String.valueOf(metricsData.getId()); - String table = metricsData.getApp() + "_" + metricsData.getMetrics(); + //表名添加monitorId区分 + String table = metricsData.getApp() + "_" + metricsData.getMetrics()+ "_" +monitorId; TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); List semanticTypes = new LinkedList<>(Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp)); @@ -114,6 +113,7 @@ void saveData(CollectRep.MetricsData metricsData) { List fieldsList = metricsData.getFieldsList(); for (CollectRep.Field field : fieldsList) { + log.info("Field " + field.getName()); semanticTypes.add(SemanticType.Field); columnNames.add(field.getName()); // handle field type @@ -171,14 +171,55 @@ public Map> getHistoryMetricData(Long monitorId, String app, Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config IotDb--------------\n" + + "\t--------------Please Config GrepTime--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } + + String[] numberAndTime = history.split("(?<=\\D)(?=\\d)|(?<=\\d)(?=\\D)"); + if (Objects.equals(numberAndTime[1], "h")){ + history = "'"+numberAndTime[0]+"'" +" HOUR"; + } + String table = app + "_" + metrics + "_" + monitorId; + String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); + log.info("selectSql: {}", selectSql); + QueryRequest request = QueryRequest.newBuilder() // + .exprType(SelectExprType.Sql) // + .ql(selectSql) // + .build(); + Result result = null; + try { + CompletableFuture> future = greptimeDb.query(request); + result = future.get(); + } catch (InterruptedException | ExecutionException e) { + log.error(e.getMessage()); + } + + if (result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + List> maps = rows.collectToMaps(); + List valueList; + for (Map map : maps) { + log.info("Query row: {}", map); + String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(strValue, (long)map.get("ts"))); + + + } + } else { + log.error("Failed to query: {}", result.getErr()); + } + return instanceValuesMap; } + + + + @Override public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { From 3c0e487643104daa8cd531333ef5b2a70154501e Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Thu, 30 Mar 2023 11:29:33 +0800 Subject: [PATCH 13/28] [warehouse]feature: 1.add GrepTimeDb `createDatabase` 2.Check if the database exists; if it does not exist, create the database. --- .../store/HistoryGrepTimeDbDataStorage.java | 81 +++++++++++++++---- 1 file changed, 65 insertions(+), 16 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index e73f32c3c72..a17a01107e1 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -23,6 +23,7 @@ import com.usthe.warehouse.config.WarehouseProperties; import io.greptime.models.*; import lombok.extern.slf4j.Slf4j; +import org.apache.arrow.flight.FlightRuntimeException; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -66,7 +67,9 @@ public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= now() - INTERVAL %s order by ts desc;"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = "SELECT FIRST_VALUE(%s), AVG(%s), MIN_VALUE(%s), MAX_VALUE(%s) FROM %s GROUP BY ([now() - %s, now()), 4h) WITHOUT NULL ANY"; - + private static final String TABLE_NOT_EXIST = "not exist"; + private static final String DATABASE_NOT_EXIST = "not exist"; + private static final String HERTZBEAT_DB_NAME = "hertzbeat"; private GreptimeDB greptimeDb; public HistoryGrepTimeDbDataStorage(WarehouseProperties properties) { @@ -87,9 +90,56 @@ private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProper } return createDatabase(); } + // 检查数据库是否存在；如果不存在，则创建该数据库 private boolean createDatabase() { - // todo auto create database hertzbeat - return true; + // 查询现有数据库 + QueryRequest showDatabases = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql("SHOW DATABASES;") + .build(); + Result result = null; + try { + CompletableFuture> future = greptimeDb.query(showDatabases); + result = future.get(); + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && !msg.contains(DATABASE_NOT_EXIST)) { + log.warn(msg); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + // 检查现有数据库是否包括“hertzbeat” + boolean isDatabaseExist = false; + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + List rowsList = rows.collect(); + for (Row row : rowsList) { + for (io.greptime.models.Value value : row.values()) { + if (value.value().toString().equals(HERTZBEAT_DB_NAME)) { + log.info("Exist Database {}",HERTZBEAT_DB_NAME); + isDatabaseExist = true; + break; + } + } + } + } + // 如果“hertzbeat”数据库不存在，则创建该数据库 + if (!isDatabaseExist) { + QueryRequest createDatabase = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql("CREATE DATABASE " + HERTZBEAT_DB_NAME + ";") + .build(); + try { + CompletableFuture> createFuture = greptimeDb.query(createDatabase); + isDatabaseExist = createFuture.get().isOk(); + log.info("Database {} does not exist,and has been created",HERTZBEAT_DB_NAME); + } catch (InterruptedException | ExecutionException e) { + log.error("Error creating database"); + } + } + return isDatabaseExist; } @Override @@ -113,7 +163,6 @@ void saveData(CollectRep.MetricsData metricsData) { List fieldsList = metricsData.getFieldsList(); for (CollectRep.Field field : fieldsList) { - log.info("Field " + field.getName()); semanticTypes.add(SemanticType.Field); columnNames.add(field.getName()); // handle field type @@ -183,33 +232,33 @@ public Map> getHistoryMetricData(Long monitorId, String app, String table = app + "_" + metrics + "_" + monitorId; String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); log.info("selectSql: {}", selectSql); - QueryRequest request = QueryRequest.newBuilder() // - .exprType(SelectExprType.Sql) // - .ql(selectSql) // + QueryRequest request = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql(selectSql) .build(); Result result = null; try { CompletableFuture> future = greptimeDb.query(request); result = future.get(); - } catch (InterruptedException | ExecutionException e) { - log.error(e.getMessage()); + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && !msg.contains(TABLE_NOT_EXIST)) { + log.warn(msg); + } + } catch (Exception e) { + log.error(e.getMessage(), e); } - if (result.isOk()) { + if (result != null && result.isOk()) { QueryOk queryOk = result.getOk(); SelectRows rows = queryOk.getRows(); List> maps = rows.collectToMaps(); List valueList; for (Map map : maps) { - log.info("Query row: {}", map); String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); - valueList.add(new Value(strValue, (long)map.get("ts"))); - - + valueList.add(new Value(strValue, (long) map.get("ts"))); } - } else { - log.error("Failed to query: {}", result.getErr()); } return instanceValuesMap; From 7f30163d2120f04553932c74f7bbc92c007b4aa1 Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Fri, 31 Mar 2023 09:26:42 +0800 Subject: [PATCH 14/28] [warehouse]feature: 1.add GrepTimeDb `getHistory` --- .../store/HistoryGrepTimeDbDataStorage.java | 91 +++++++++++++++---- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index a17a01107e1..f956b10a4aa 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -22,6 +22,7 @@ import com.usthe.common.util.CommonConstants; import com.usthe.warehouse.config.WarehouseProperties; import io.greptime.models.*; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.flight.FlightRuntimeException; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -33,6 +34,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.LocalDateTime; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -66,10 +68,9 @@ public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { private static final String QUERY_HISTORY_SQL = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= now() - INTERVAL %s order by ts desc;"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL - = "SELECT FIRST_VALUE(%s), AVG(%s), MIN_VALUE(%s), MAX_VALUE(%s) FROM %s GROUP BY ([now() - %s, now()), 4h) WITHOUT NULL ANY"; + = "SELECT first(%s), avg(%s), min(%s), max(%s) FROM %s WHERE instance = %s AND ts >= now - %s INTERVAL '4' HOUR"; private static final String TABLE_NOT_EXIST = "not exist"; private static final String DATABASE_NOT_EXIST = "not exist"; - private static final String HERTZBEAT_DB_NAME = "hertzbeat"; private GreptimeDB greptimeDb; public HistoryGrepTimeDbDataStorage(WarehouseProperties properties) { @@ -117,8 +118,8 @@ private boolean createDatabase() { List rowsList = rows.collect(); for (Row row : rowsList) { for (io.greptime.models.Value value : row.values()) { - if (value.value().toString().equals(HERTZBEAT_DB_NAME)) { - log.info("Exist Database {}",HERTZBEAT_DB_NAME); + if (value.value().toString().equals(STORAGE_DATABASE)) { + log.info("Exist Database {}",STORAGE_DATABASE); isDatabaseExist = true; break; } @@ -129,12 +130,12 @@ private boolean createDatabase() { if (!isDatabaseExist) { QueryRequest createDatabase = QueryRequest.newBuilder() .exprType(SelectExprType.Sql) - .ql("CREATE DATABASE " + HERTZBEAT_DB_NAME + ";") + .ql("CREATE DATABASE " + STORAGE_DATABASE + ";") .build(); try { CompletableFuture> createFuture = greptimeDb.query(createDatabase); isDatabaseExist = createFuture.get().isOk(); - log.info("Database {} does not exist,and has been created",HERTZBEAT_DB_NAME); + log.info("Database {} does not exist,and has been created",STORAGE_DATABASE); } catch (InterruptedException | ExecutionException e) { log.error("Error creating database"); } @@ -151,10 +152,10 @@ void saveData(CollectRep.MetricsData metricsData) { log.info("[warehouse greptime] flush metrics data {} is null, ignore.", metricsData.getId()); return; } - String monitorId = String.valueOf(metricsData.getId()); //表名添加monitorId区分 String table = metricsData.getApp() + "_" + metricsData.getMetrics()+ "_" +monitorId; + //TODO bug：选择STORAGE_DATABASE不起作用，还是默认存在public里 TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); List semanticTypes = new LinkedList<>(Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp)); @@ -224,11 +225,7 @@ public Map> getHistoryMetricData(Long monitorId, String app, "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } - - String[] numberAndTime = history.split("(?<=\\D)(?=\\d)|(?<=\\d)(?=\\D)"); - if (Objects.equals(numberAndTime[1], "h")){ - history = "'"+numberAndTime[0]+"'" +" HOUR"; - } + history = getHistory(history); String table = app + "_" + metrics + "_" + monitorId; String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); log.info("selectSql: {}", selectSql); @@ -243,6 +240,9 @@ public Map> getHistoryMetricData(Long monitorId, String app, } catch (FlightRuntimeException e) { String msg = e.getMessage(); if (msg != null && !msg.contains(TABLE_NOT_EXIST)) { + List valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(null, System.currentTimeMillis())); + log.info("TABLE_NOT_EXIST: {}",valueList); log.warn(msg); } } catch (Exception e) { @@ -260,27 +260,80 @@ public Map> getHistoryMetricData(Long monitorId, String app, valueList.add(new Value(strValue, (long) map.get("ts"))); } } - + log.info("instanceValuesMap: {}",instanceValuesMap); return instanceValuesMap; } - - - - - @Override public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config IotDb--------------\n" + + "\t--------------Please Config GrepTime--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } + history = getHistory(history); + String table = app + "_" + metrics + "_" + monitorId; + String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); + log.info("selectSql: {}", selectSql); + QueryRequest request = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql(selectSql) + .build(); + Result result = null; + try { + CompletableFuture> future = greptimeDb.query(request); + result = future.get(); + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && !msg.contains(TABLE_NOT_EXIST)) { + log.warn(msg); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + List valueList; + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + List> maps = rows.collectToMaps(); + for (Map map : maps) { + String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(strValue, (long) map.get("ts"))); + } + } return instanceValuesMap; } + private static String getHistory(String history) { + String[] parts = history.split("(?<=\\D)(?=\\d)|(?<=\\d)(?=\\D)"); + String number = parts[0]; + String time = parts[1]; + switch (time) { + case "h": + time = "HOUR"; + break; + case "d": + time = "DAY"; + break; + case "w": + time = "WEEK"; + break; + case "s": + time = "SECOND"; + break; + case "m": + time = "MINUTE"; + break; + default: + number = "6"; + time = "HOUR"; + break; + } + return String.format("'%s' %s", number, time); + } @Override public void destroy() { From cfdb92de56e74185e6c827b49438590f43ccf558 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Fri, 31 Mar 2023 18:58:34 +0800 Subject: [PATCH 15/28] [warehouse] bugfix can not catch exception when table not exist --- .../store/HistoryGrepTimeDbDataStorage.java | 74 ++++--------------- 1 file changed, 16 insertions(+), 58 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index f956b10a4aa..30acd23f090 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -228,84 +228,42 @@ public Map> getHistoryMetricData(Long monitorId, String app, history = getHistory(history); String table = app + "_" + metrics + "_" + monitorId; String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); - log.info("selectSql: {}", selectSql); + log.debug("selectSql: {}", selectSql); QueryRequest request = QueryRequest.newBuilder() .exprType(SelectExprType.Sql) .ql(selectSql) .build(); - Result result = null; try { CompletableFuture> future = greptimeDb.query(request); - result = future.get(); + Result result = future.get(); + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + List> maps = rows.collectToMaps(); + List valueList; + for (Map map : maps) { + String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(strValue, (long) map.get("ts"))); + } + } } catch (FlightRuntimeException e) { String msg = e.getMessage(); - if (msg != null && !msg.contains(TABLE_NOT_EXIST)) { + if (msg != null && msg.contains(TABLE_NOT_EXIST)) { List valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); valueList.add(new Value(null, System.currentTimeMillis())); - log.info("TABLE_NOT_EXIST: {}",valueList); - log.warn(msg); + log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); } } catch (Exception e) { log.error(e.getMessage(), e); } - - if (result != null && result.isOk()) { - QueryOk queryOk = result.getOk(); - SelectRows rows = queryOk.getRows(); - List> maps = rows.collectToMaps(); - List valueList; - for (Map map : maps) { - String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); - valueList.add(new Value(strValue, (long) map.get("ts"))); - } - } - log.info("instanceValuesMap: {}",instanceValuesMap); return instanceValuesMap; } @Override public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { - Map> instanceValuesMap = new HashMap<>(8); - if (!isServerAvailable()) { - log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config GrepTime--------------\n" + - "\t----------Can Not Use Metric History Now----------\n"); - return instanceValuesMap; - } - history = getHistory(history); - String table = app + "_" + metrics + "_" + monitorId; - String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); - log.info("selectSql: {}", selectSql); - QueryRequest request = QueryRequest.newBuilder() - .exprType(SelectExprType.Sql) - .ql(selectSql) - .build(); - Result result = null; - try { - CompletableFuture> future = greptimeDb.query(request); - result = future.get(); - } catch (FlightRuntimeException e) { - String msg = e.getMessage(); - if (msg != null && !msg.contains(TABLE_NOT_EXIST)) { - log.warn(msg); - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } - List valueList; - if (result != null && result.isOk()) { - QueryOk queryOk = result.getOk(); - SelectRows rows = queryOk.getRows(); - List> maps = rows.collectToMaps(); - for (Map map : maps) { - String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); - valueList.add(new Value(strValue, (long) map.get("ts"))); - } - } - return instanceValuesMap; + return null; } private static String getHistory(String history) { String[] parts = history.split("(?<=\\D)(?=\\d)|(?<=\\d)(?=\\D)"); From e387ba85db74e2aa11b483e2e0c3455f6157f184 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Fri, 31 Mar 2023 19:47:20 +0800 Subject: [PATCH 16/28] [warehouse] support time period string query, 1h 3D 3W --- .../store/HistoryGrepTimeDbDataStorage.java | 50 +-- .../store/HistoryJpaDatabaseDataStorage.java | 365 +++++++++--------- 2 files changed, 199 insertions(+), 216 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index 30acd23f090..eed4d46c60a 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -20,13 +20,11 @@ import com.usthe.common.entity.dto.Value; import com.usthe.common.entity.message.CollectRep; import com.usthe.common.util.CommonConstants; +import com.usthe.common.util.TimePeriodUtil; import com.usthe.warehouse.config.WarehouseProperties; import io.greptime.models.*; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.flight.FlightRuntimeException; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import io.greptime.GreptimeDB; @@ -34,7 +32,9 @@ import java.math.BigDecimal; import java.math.RoundingMode; -import java.time.LocalDateTime; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -66,7 +66,7 @@ public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { private static final String SHOW_STORAGE_GROUP = "show storage group"; private static final String QUERY_HISTORY_SQL - = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= now() - INTERVAL %s order by ts desc;"; + = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= %s order by ts desc;"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = "SELECT first(%s), avg(%s), min(%s), max(%s) FROM %s WHERE instance = %s AND ts >= now - %s INTERVAL '4' HOUR"; private static final String TABLE_NOT_EXIST = "not exist"; @@ -225,9 +225,18 @@ public Map> getHistoryMetricData(Long monitorId, String app, "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } - history = getHistory(history); + long expireTime = 0; + try { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + expireTime = dateTime.toEpochSecond() * 1000; + } catch (Exception e) { + log.error("parse history time error: {}. use default: 6h", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6)); + expireTime = dateTime.toEpochSecond() * 1000; + } String table = app + "_" + metrics + "_" + monitorId; - String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history); + String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, expireTime); log.debug("selectSql: {}", selectSql); QueryRequest request = QueryRequest.newBuilder() .exprType(SelectExprType.Sql) @@ -265,33 +274,6 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str String metric, String instance, String history) { return null; } - private static String getHistory(String history) { - String[] parts = history.split("(?<=\\D)(?=\\d)|(?<=\\d)(?=\\D)"); - String number = parts[0]; - String time = parts[1]; - switch (time) { - case "h": - time = "HOUR"; - break; - case "d": - time = "DAY"; - break; - case "w": - time = "WEEK"; - break; - case "s": - time = "SECOND"; - break; - case "m": - time = "MINUTE"; - break; - default: - number = "6"; - time = "HOUR"; - break; - } - return String.format("'%s' %s", number, time); - } @Override public void destroy() { diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java index a48a571e9f3..48e24c35eab 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryJpaDatabaseDataStorage.java @@ -49,196 +49,197 @@ */ @Component @ConditionalOnProperty(prefix = "warehouse.store.jpa", - name = "enabled", havingValue = "true") + name = "enabled", havingValue = "true") @Slf4j public class HistoryJpaDatabaseDataStorage extends AbstractHistoryDataStorage { - private HistoryDao historyDao; - private WarehouseProperties.StoreProperties.JpaProperties jpaProperties; + private HistoryDao historyDao; + private WarehouseProperties.StoreProperties.JpaProperties jpaProperties; - private static final int STRING_MAX_LENGTH = 1024; - private static final int MAX_HISTORY_TABLE_RECORD = 60_000; + private static final int STRING_MAX_LENGTH = 1024; + private static final int MAX_HISTORY_TABLE_RECORD = 60_000; - public HistoryJpaDatabaseDataStorage(WarehouseProperties properties, - HistoryDao historyDao) { - this.jpaProperties = properties.getStore().getJpa(); - this.serverAvailable = true; - this.historyDao = historyDao; - } + public HistoryJpaDatabaseDataStorage(WarehouseProperties properties, + HistoryDao historyDao) { + this.jpaProperties = properties.getStore().getJpa(); + this.serverAvailable = true; + this.historyDao = historyDao; + } - @Scheduled( fixedDelay = 1, timeUnit = TimeUnit.MINUTES) - public void expiredDataCleaner() { - log.warn("[jpa-metrics-store]-start running expired data cleaner." + - "Please use time series db instead of jpa for better performance"); - String expireTimeStr = jpaProperties.getExpireTime(); - long expireTime = 0; - try { - if (NumberUtils.isParsable(expireTimeStr)) { - expireTime = NumberUtils.toLong(expireTimeStr); - expireTime = (ZonedDateTime.now().toEpochSecond() + expireTime) * 1000; - } else { - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(expireTimeStr); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - expireTime = dateTime.toEpochSecond() * 1000; - } - } catch (Exception e) { - log.error("expiredDataCleaner time error: {}. use default expire time to clean: 1h", e.getMessage()); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(1)); - expireTime = dateTime.toEpochSecond() * 1000; - } - try { - int rows = historyDao.deleteHistoriesByTimeBefore(expireTime); - log.info("[jpa-metrics-store]-delete {} rows.", rows); - long total = historyDao.count(); - if (total > MAX_HISTORY_TABLE_RECORD) { - rows = historyDao.deleteOlderHistoriesRecord(); - log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows); - } - } catch (Exception e) { - log.error("expiredDataCleaner database error: {}.", e.getMessage()); - log.error("try to truncate table hzb_history. Please use time series db instead of jpa for better performance."); - historyDao.truncateTable(); - } - } + @Scheduled(fixedDelay = 1, timeUnit = TimeUnit.MINUTES) + public void expiredDataCleaner() { + log.warn("[jpa-metrics-store]-start running expired data cleaner." + + "Please use time series db instead of jpa for better performance"); + String expireTimeStr = jpaProperties.getExpireTime(); + long expireTime = 0; + try { + if (NumberUtils.isParsable(expireTimeStr)) { + expireTime = NumberUtils.toLong(expireTimeStr); + expireTime = (ZonedDateTime.now().toEpochSecond() + expireTime) * 1000; + } else { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(expireTimeStr); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + expireTime = dateTime.toEpochSecond() * 1000; + } + } catch (Exception e) { + log.error("expiredDataCleaner time error: {}. use default expire time to clean: 1h", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(1)); + expireTime = dateTime.toEpochSecond() * 1000; + } + try { + int rows = historyDao.deleteHistoriesByTimeBefore(expireTime); + log.info("[jpa-metrics-store]-delete {} rows.", rows); + long total = historyDao.count(); + if (total > MAX_HISTORY_TABLE_RECORD) { + rows = historyDao.deleteOlderHistoriesRecord(); + log.warn("[jpa-metrics-store]-force delete {} rows due too many. Please use time series db instead of jpa for better performance.", rows); + } + } catch (Exception e) { + log.error("expiredDataCleaner database error: {}.", e.getMessage()); + log.error("try to truncate table hzb_history. Please use time series db instead of jpa for better performance."); + historyDao.truncateTable(); + } + } - @Override - void saveData(CollectRep.MetricsData metricsData) { - if (metricsData.getCode() != CollectRep.Code.SUCCESS) { - return; - } - if (metricsData.getValuesList().isEmpty()) { - log.info("[warehouse jpa] flush metrics data {} is null, ignore.", metricsData.getId()); - return; - } - String monitorType = metricsData.getApp(); - String metrics = metricsData.getMetrics(); - List fieldsList = metricsData.getFieldsList(); - try { - List historyList = new LinkedList<>(); - History.HistoryBuilder historyBuilder = History.builder() - .monitorId(metricsData.getId()) - .app(monitorType) - .metrics(metrics) - .time(metricsData.getTime()); - for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { - String instance = valueRow.getInstance(); - if (!instance.isEmpty()) { - instance = formatStrValue(instance); - historyBuilder.instance(instance); - } else { - historyBuilder.instance(null); - } - for (int i = 0; i < fieldsList.size(); i++) { - CollectRep.Field field = fieldsList.get(i); - // ignore string value store in db - if (field.getType() == CommonConstants.TYPE_STRING) { - continue; - } - historyBuilder.metric(field.getName()); - if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { - if (field.getType() == CommonConstants.TYPE_NUMBER) { - historyBuilder.metricType(CommonConstants.TYPE_NUMBER) - .dou(Double.parseDouble(valueRow.getColumns(i))); - } else if (field.getType() == CommonConstants.TYPE_STRING) { - historyBuilder.metricType(CommonConstants.TYPE_STRING) - .str(formatStrValue(valueRow.getColumns(i))); - } - } else { - if (field.getType() == CommonConstants.TYPE_NUMBER) { - historyBuilder.metricType(CommonConstants.TYPE_NUMBER).dou(null); - } else if (field.getType() == CommonConstants.TYPE_STRING) { - historyBuilder.metricType(CommonConstants.TYPE_STRING).str(null); - } - } - historyList.add(historyBuilder.build()); - } - } - historyDao.saveAll(historyList); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } + @Override + void saveData(CollectRep.MetricsData metricsData) { + if (metricsData.getCode() != CollectRep.Code.SUCCESS) { + return; + } + if (metricsData.getValuesList().isEmpty()) { + log.info("[warehouse jpa] flush metrics data {} is null, ignore.", metricsData.getId()); + return; + } + String monitorType = metricsData.getApp(); + String metrics = metricsData.getMetrics(); + List fieldsList = metricsData.getFieldsList(); + try { + List historyList = new LinkedList<>(); + History.HistoryBuilder historyBuilder = History.builder() + .monitorId(metricsData.getId()) + .app(monitorType) + .metrics(metrics) + .time(metricsData.getTime()); + for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + String instance = valueRow.getInstance(); + if (!instance.isEmpty()) { + instance = formatStrValue(instance); + historyBuilder.instance(instance); + } else { + historyBuilder.instance(null); + } + for (int i = 0; i < fieldsList.size(); i++) { + CollectRep.Field field = fieldsList.get(i); + // ignore string value store in db + if (field.getType() == CommonConstants.TYPE_STRING) { + continue; + } + historyBuilder.metric(field.getName()); + if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { + if (field.getType() == CommonConstants.TYPE_NUMBER) { + historyBuilder.metricType(CommonConstants.TYPE_NUMBER) + .dou(Double.parseDouble(valueRow.getColumns(i))); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + historyBuilder.metricType(CommonConstants.TYPE_STRING) + .str(formatStrValue(valueRow.getColumns(i))); + } + } else { + if (field.getType() == CommonConstants.TYPE_NUMBER) { + historyBuilder.metricType(CommonConstants.TYPE_NUMBER).dou(null); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + historyBuilder.metricType(CommonConstants.TYPE_STRING).str(null); + } + } + historyList.add(historyBuilder.build()); + } + } + historyDao.saveAll(historyList); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } - /** - * 从数据库获取指标历史数据 - * - * @param monitorId 监控ID - * @param app 监控类型 - * @param metrics 指标集合名 - * @param metric 指标名 - * @param instance 实例 - * @param history 历史范围 - * @return 指标历史数据列表 - */ - @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { - Map> instanceValuesMap = new HashMap<>(8); - Specification specification = (root, query, criteriaBuilder) -> { - List andList = new ArrayList<>(); - Predicate predicateMonitorId = criteriaBuilder.equal(root.get("monitorId"), monitorId); - Predicate predicateMonitorType = criteriaBuilder.equal(root.get("app"), app); - Predicate predicateMonitorMetrics = criteriaBuilder.equal(root.get("metrics"), metrics); - Predicate predicateMonitorMetric = criteriaBuilder.equal(root.get("metric"), metric); - andList.add(predicateMonitorId); - andList.add(predicateMonitorType); - andList.add(predicateMonitorMetrics); - andList.add(predicateMonitorMetric); - if (instance != null && !"".equals(instance)) { - Predicate predicateMonitorInstance = criteriaBuilder.equal(root.get("instance"), instance); - andList.add(predicateMonitorInstance); - } - if (history != null) { - try { - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - long timeBefore = dateTime.toEpochSecond() * 1000; - Predicate timePredicate = criteriaBuilder.ge(root.get("time"), timeBefore); - andList.add(timePredicate); - } catch (Exception e) { - log.error(e.getMessage()); - } - } - Predicate[] predicates = new Predicate[andList.size()]; - Predicate predicate = criteriaBuilder.and(andList.toArray(predicates)); - return query.where(predicate).getRestriction(); - }; - Sort sortExp = Sort.by(new Sort.Order(Sort.Direction.DESC, "time")); - List historyList = historyDao.findAll(specification, sortExp); - for (History dataItem : historyList) { - String value = ""; - if (dataItem.getMetricType() == CommonConstants.TYPE_NUMBER) { - if (dataItem.getDou() != null) { - value = BigDecimal.valueOf(dataItem.getDou()).setScale(4, RoundingMode.HALF_UP) - .stripTrailingZeros().toPlainString(); - } - } else { - value = dataItem.getStr(); - } - String instanceValue = dataItem.getInstance() == null ? "" : dataItem.getInstance(); - List valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); - valueList.add(new Value(value, dataItem.getTime())); - } - return instanceValuesMap; - } + /** + * 从数据库获取指标历史数据 + * + * @param monitorId 监控ID + * @param app 监控类型 + * @param metrics 指标集合名 + * @param metric 指标名 + * @param instance 实例 + * @param history 历史范围 + * @return 指标历史数据列表 + */ + @Override + public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { + Map> instanceValuesMap = new HashMap<>(8); + Specification specification = (root, query, criteriaBuilder) -> { + List andList = new ArrayList<>(); + Predicate predicateMonitorId = criteriaBuilder.equal(root.get("monitorId"), monitorId); + Predicate predicateMonitorType = criteriaBuilder.equal(root.get("app"), app); + Predicate predicateMonitorMetrics = criteriaBuilder.equal(root.get("metrics"), metrics); + Predicate predicateMonitorMetric = criteriaBuilder.equal(root.get("metric"), metric); + andList.add(predicateMonitorId); + andList.add(predicateMonitorType); + andList.add(predicateMonitorMetrics); + andList.add(predicateMonitorMetric); + if (instance != null && !"".equals(instance)) { + Predicate predicateMonitorInstance = criteriaBuilder.equal(root.get("instance"), instance); + andList.add(predicateMonitorInstance); + } + if (history != null) { + try { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + long timeBefore = dateTime.toEpochSecond() * 1000; + Predicate timePredicate = criteriaBuilder.ge(root.get("time"), timeBefore); + andList.add(timePredicate); + } catch (Exception e) { + log.error(e.getMessage()); + } + } + Predicate[] predicates = new Predicate[andList.size()]; + Predicate predicate = criteriaBuilder.and(andList.toArray(predicates)); + return query.where(predicate).getRestriction(); + }; + Sort sortExp = Sort.by(new Sort.Order(Sort.Direction.DESC, "time")); + List historyList = historyDao.findAll(specification, sortExp); + for (History dataItem : historyList) { + String value = ""; + if (dataItem.getMetricType() == CommonConstants.TYPE_NUMBER) { + if (dataItem.getDou() != null) { + value = BigDecimal.valueOf(dataItem.getDou()).setScale(4, RoundingMode.HALF_UP) + .stripTrailingZeros().toPlainString(); + } + } else { + value = dataItem.getStr(); + } + String instanceValue = dataItem.getInstance() == null ? "" : dataItem.getInstance(); + List valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); + valueList.add(new Value(value, dataItem.getTime())); + } + return instanceValuesMap; + } - private String formatStrValue(String value) { - if (value == null) { - return ""; - } - value = value.replace("'", "\\'"); - value = value.replace("\"", "\\\""); - value = value.replace("*", "-"); - value = String.format("`%s`", value); - if (value.length() > STRING_MAX_LENGTH) { - value = value.substring(0, STRING_MAX_LENGTH - 1); - } - return value; - } + private String formatStrValue(String value) { + if (value == null) { + return ""; + } + value = value.replace("'", "\\'"); + value = value.replace("\"", "\\\""); + value = value.replace("*", "-"); + value = String.format("`%s`", value); + if (value.length() > STRING_MAX_LENGTH) { + value = value.substring(0, STRING_MAX_LENGTH - 1); + } + return value; + } - @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { - return new HashMap<>(8); - } + @Override + public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { + return new HashMap<>(8); + } - @Override - public void destroy() throws Exception {} + @Override + public void destroy() throws Exception { + } } From eb26895bb56ed80a0c99d012a65d157e66d0fdd6 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Fri, 31 Mar 2023 20:01:29 +0800 Subject: [PATCH 17/28] [warehouse] support time period string query, 1h 3D 3W --- .../store/HistoryGrepTimeDbDataStorage.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index eed4d46c60a..0f4c0ab48e5 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -40,10 +40,10 @@ import java.util.concurrent.ExecutionException; /** - * IoTDB data storage + * greptimeDB data storage * - * @author ceilzcx - * @since 2022/10/12 + * @author zqr10159 tomsun28 + * @since 2023/03/25 */ @Component @ConditionalOnProperty(prefix = "warehouse.store.greptime", @@ -55,16 +55,6 @@ public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { * storage database */ private static final String STORAGE_DATABASE = "hertzbeat"; - private static final String INSTANCE ="instance"; - - private static final String SET_TTL = "set ttl to %s %s"; - - private static final String CANCEL_TTL = "unset ttl to %s"; - - private static final String SHOW_DEVICES = "SHOW DEVICES %s"; - - private static final String SHOW_STORAGE_GROUP = "show storage group"; - private static final String QUERY_HISTORY_SQL = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= %s order by ts desc;"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL @@ -91,7 +81,11 @@ private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProper } return createDatabase(); } - // 检查数据库是否存在；如果不存在，则创建该数据库 + + /** + * Check if the database exists; If it does not exist, the database is created + * 检查数据库是否存在；如果不存在，则创建该数据库 + */ private boolean createDatabase() { // 查询现有数据库 QueryRequest showDatabases = QueryRequest.newBuilder() @@ -110,6 +104,7 @@ private boolean createDatabase() { } catch (Exception e) { log.error(e.getMessage(), e); } + // Check if the database exists; // 检查现有数据库是否包括“hertzbeat” boolean isDatabaseExist = false; if (result != null && result.isOk()) { @@ -126,6 +121,7 @@ private boolean createDatabase() { } } } + // If it does not exist, create database // 如果“hertzbeat”数据库不存在，则创建该数据库 if (!isDatabaseExist) { QueryRequest createDatabase = QueryRequest.newBuilder() From 6e56783c034aaadccc35b5a4d1901bb70caddee3 Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Sun, 2 Apr 2023 21:28:43 +0800 Subject: [PATCH 18/28] [warehouse]feature: 1.add GrepTimeDb `getHistoryIntervalMetricData` --- .../store/HistoryGrepTimeDbDataStorage.java | 152 ++++++++++++++++-- 1 file changed, 140 insertions(+), 12 deletions(-) diff --git a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java index 0f4c0ab48e5..32f2b7c81a0 100644 --- a/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/com/usthe/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -29,15 +29,19 @@ import org.springframework.stereotype.Component; import io.greptime.GreptimeDB; import io.greptime.options.GreptimeOptions; - import java.math.BigDecimal; import java.math.RoundingMode; +import java.sql.Timestamp; import java.time.Duration; +import java.time.OffsetDateTime; +import java.time.OffsetTime; import java.time.ZonedDateTime; import java.time.temporal.TemporalAmount; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * greptimeDB data storage @@ -56,9 +60,11 @@ public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { */ private static final String STORAGE_DATABASE = "hertzbeat"; private static final String QUERY_HISTORY_SQL - = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= %s order by ts desc;"; + = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= %s and monitor_id = %s order by ts desc;"; + private static final String QUERY_INSTANCE_SQL + = "SELECT DISTINCT instance FROM %s WHERE ts >= now() - interval '1' WEEK"; private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL - = "SELECT first(%s), avg(%s), min(%s), max(%s) FROM %s WHERE instance = %s AND ts >= now - %s INTERVAL '4' HOUR"; + = "SELECT first, avg ,max, min FROM (SELECT %s as first FROM %s WHERE monitor_id = %s and ts >= %s and ts < %s ORDER BY ts LIMIT 1) LEFT JOIN (SELECT avg(%s) as avg, min(%s) as min, max(%s) as max FROM %s WHERE ts >= %s and ts < %s) ON 1=1"; private static final String TABLE_NOT_EXIST = "not exist"; private static final String DATABASE_NOT_EXIST = "not exist"; private GreptimeDB greptimeDb; @@ -83,7 +89,8 @@ private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProper } /** - * Check if the database exists; If it does not exist, the database is created + * + * Checks if the database exists; if not, creates the Database. * 检查数据库是否存在；如果不存在，则创建该数据库 */ private boolean createDatabase() { @@ -96,13 +103,13 @@ private boolean createDatabase() { try { CompletableFuture> future = greptimeDb.query(showDatabases); result = future.get(); - } catch (FlightRuntimeException e) { + } catch (Exception e) { + log.info("TABLE_NOT_EXIST: {}",e.getMessage()); String msg = e.getMessage(); if (msg != null && !msg.contains(DATABASE_NOT_EXIST)) { log.warn(msg); } - } catch (Exception e) { - log.error(e.getMessage(), e); + } // Check if the database exists; // 检查现有数据库是否包括“hertzbeat” @@ -149,8 +156,7 @@ void saveData(CollectRep.MetricsData metricsData) { return; } String monitorId = String.valueOf(metricsData.getId()); - //表名添加monitorId区分 - String table = metricsData.getApp() + "_" + metricsData.getMetrics()+ "_" +monitorId; + String table = metricsData.getApp() + "_" + metricsData.getMetrics(); //TODO bug：选择STORAGE_DATABASE不起作用，还是默认存在public里 TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); @@ -231,8 +237,8 @@ public Map> getHistoryMetricData(Long monitorId, String app, ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6)); expireTime = dateTime.toEpochSecond() * 1000; } - String table = app + "_" + metrics + "_" + monitorId; - String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, expireTime); + String table = app + "_" + metrics; + String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, expireTime,monitorId); log.debug("selectSql: {}", selectSql); QueryRequest request = QueryRequest.newBuilder() .exprType(SelectExprType.Sql) @@ -268,7 +274,129 @@ public Map> getHistoryMetricData(Long monitorId, String app, @Override public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { - return null; + Map> instanceValuesMap = new HashMap<>(8); + if (!isServerAvailable()) { + log.error("\n\t---------------GrepTime Init Failed---------------\n" + + "\t--------------Please Config GrepTime--------------\n" + + "\t----------Can Not Use Metric History Now----------\n"); + return instanceValuesMap; + } + String table = app + "_" + metrics; + List instances = new LinkedList<>(); + if (instance != null) { + instances.add(instance); + } + if (instances.isEmpty()){ + String selectSql = String.format(QUERY_INSTANCE_SQL, table); + log.debug("selectSql: {}", selectSql); + QueryRequest request = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql(selectSql) + .build(); + try { + CompletableFuture> future = greptimeDb.query(request); + Result result = future.get(); + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + while (rows.hasNext()){ + Row row = rows.next(); + if (row !=null){ + List values = row.values(); + for (io.greptime.models.Value value : values){ + log.info("value:{}", value.value()); + Object instanceValue = value.value(); + if (instanceValue == null || "".equals(instanceValue)) { + instances.add("''"); + } else { + instances.add(instanceValue.toString()); + } + } + } + + } + } + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && msg.contains(TABLE_NOT_EXIST)) { + log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + long startTime; + long endTime; + try { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + startTime = dateTime.toEpochSecond() * 1000; + } catch (Exception e) { + log.error("parse history time error: {}. use default: 6h", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6)); + startTime = dateTime.toEpochSecond() * 1000; + } + + Calendar cal = Calendar.getInstance(); + + long interval = System.currentTimeMillis() - startTime; + long fourHourCount = TimeUnit.MILLISECONDS.toHours(interval) / 4; + for (int i = 0; i < fourHourCount; i++){ + cal.clear(); + cal.setTimeInMillis(startTime); + cal.add(Calendar.HOUR_OF_DAY, 4); + endTime = cal.getTimeInMillis(); + + for (String instanceValue:instances){ + String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, table, monitorId, startTime, endTime, metric, metric, metric, table, startTime, endTime); + + log.info("selectSql: {}", selectSql); + QueryRequest request = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql(selectSql) + .build(); + List values = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); + try { + CompletableFuture> future = greptimeDb.query(request); + Result result = future.get(); + log.info("result:{}", result); + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + String[] col = new String[4]; + while (rows.hasNext()){ + Row row = rows.next(); + if (!row.values().isEmpty()){ + for (int j = 0; j < row.values().size(); j++){ + log.info("value:{}", row.values().get(j)); + String colStr = new BigDecimal(row.values().get(j).value().toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + col[j] = colStr; + } + Value valueBuild = Value.builder() + .origin(col[0]).mean(col[1]) + .min(col[2]).max(col[3]) + .time(System.currentTimeMillis()) + .build(); + values.add(valueBuild); + } + } + log.info("values:{}", values); + } + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && msg.contains(TABLE_NOT_EXIST)) { + List valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(null, System.currentTimeMillis())); + log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + startTime = endTime; + } + + return instanceValuesMap; } @Override From 48385916b23d02612265026b7117dc73423fc0f0 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Sun, 2 Apr 2023 21:50:35 +0800 Subject: [PATCH 19/28] merge master --- .../store/HistoryGrepTimeDbDataStorage.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index 32f2b7c81a0..abc182abf7f 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -15,33 +15,29 @@ * limitations under the License. */ -package com.usthe.warehouse.store; +package org.dromara.hertzbeat.warehouse.store; -import com.usthe.common.entity.dto.Value; -import com.usthe.common.entity.message.CollectRep; -import com.usthe.common.util.CommonConstants; -import com.usthe.common.util.TimePeriodUtil; -import com.usthe.warehouse.config.WarehouseProperties; import io.greptime.models.*; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.flight.FlightRuntimeException; +import org.dromara.hertzbeat.common.entity.dto.Value; +import org.dromara.hertzbeat.common.entity.message.CollectRep; +import org.dromara.hertzbeat.common.util.CommonConstants; +import org.dromara.hertzbeat.common.util.TimePeriodUtil; +import org.dromara.hertzbeat.warehouse.config.WarehouseProperties; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import io.greptime.GreptimeDB; import io.greptime.options.GreptimeOptions; import java.math.BigDecimal; import java.math.RoundingMode; -import java.sql.Timestamp; import java.time.Duration; -import java.time.OffsetDateTime; -import java.time.OffsetTime; import java.time.ZonedDateTime; import java.time.temporal.TemporalAmount; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; /** * greptimeDB data storage From 58bcf0aad47f088df1d06ce51012c9dc180ece20 Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Sun, 2 Apr 2023 22:34:36 +0800 Subject: [PATCH 20/28] update --- .../store/HistoryGrepTimeDbDataStorage.java | 670 +++++++++--------- 1 file changed, 335 insertions(+), 335 deletions(-) diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index abc182abf7f..9f863f5ac32 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -29,6 +29,7 @@ import org.springframework.stereotype.Component; import io.greptime.GreptimeDB; import io.greptime.options.GreptimeOptions; + import java.math.BigDecimal; import java.math.RoundingMode; import java.time.Duration; @@ -47,358 +48,357 @@ */ @Component @ConditionalOnProperty(prefix = "warehouse.store.greptime", - name = "enabled", havingValue = "true") + name = "enabled", havingValue = "true") @Slf4j public class HistoryGrepTimeDbDataStorage extends AbstractHistoryDataStorage { - /** - * storage database - */ - private static final String STORAGE_DATABASE = "hertzbeat"; - private static final String QUERY_HISTORY_SQL - = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= %s and monitor_id = %s order by ts desc;"; - private static final String QUERY_INSTANCE_SQL - = "SELECT DISTINCT instance FROM %s WHERE ts >= now() - interval '1' WEEK"; - private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL - = "SELECT first, avg ,max, min FROM (SELECT %s as first FROM %s WHERE monitor_id = %s and ts >= %s and ts < %s ORDER BY ts LIMIT 1) LEFT JOIN (SELECT avg(%s) as avg, min(%s) as min, max(%s) as max FROM %s WHERE ts >= %s and ts < %s) ON 1=1"; - private static final String TABLE_NOT_EXIST = "not exist"; - private static final String DATABASE_NOT_EXIST = "not exist"; - private GreptimeDB greptimeDb; + /** + * storage database + */ + private static final String STORAGE_DATABASE = "hertzbeat"; + private static final String QUERY_HISTORY_SQL + = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= %s and monitor_id = %s order by ts desc;"; + private static final String QUERY_HISTORY_WITH_INSTANCE_SQL + = "SELECT ts, instance, \"%s\" FROM %s WHERE ts >= %s and monitor_id = %s and instance = %s order by ts desc;"; + private static final String QUERY_INSTANCE_SQL + = "SELECT DISTINCT instance FROM %s WHERE ts >= now() - interval '1' WEEK"; + private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL + = "SELECT first, avg ,max, min FROM (SELECT %s as first FROM %s WHERE monitor_id = %s and ts >= %s and ts < %s ORDER BY ts LIMIT 1) LEFT JOIN (SELECT avg(%s) as avg, min(%s) as min, max(%s) as max FROM %s WHERE ts >= %s and ts < %s) ON 1=1"; + private static final String TABLE_NOT_EXIST = "not exist"; + private static final String DATABASE_NOT_EXIST = "not exist"; + private GreptimeDB greptimeDb; + + public HistoryGrepTimeDbDataStorage(WarehouseProperties properties) { + this.serverAvailable = this.initDbSession(properties.getStore().getGreptime()); + } - public HistoryGrepTimeDbDataStorage(WarehouseProperties properties) { - this.serverAvailable = this.initDbSession(properties.getStore().getGreptime()); - } + private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProperties properties) { + String endpoint = properties.getEndpoint(); + GreptimeOptions opts = GreptimeOptions.newBuilder(endpoint) + .writeMaxRetries(1) + .readMaxRetries(2) + .routeTableRefreshPeriodSeconds(-1) + .build(); + greptimeDb = new GreptimeDB(); + if (!greptimeDb.init(opts)) { + log.error("Fail to start GreptimeDB client"); + return false; + } + return createDatabase(); + } - private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProperties properties) { - String endpoint = properties.getEndpoint(); - GreptimeOptions opts = GreptimeOptions.newBuilder(endpoint) - .writeMaxRetries(1) - .readMaxRetries(2) - .routeTableRefreshPeriodSeconds(-1) - .build(); - greptimeDb = new GreptimeDB(); - if (!greptimeDb.init(opts)) { - log.error("Fail to start GreptimeDB client"); - return false; - } - return createDatabase(); - } + /** + * Checks if the database exists; if not, creates the Database. + * 检查数据库是否存在；如果不存在，则创建该数据库 + */ + private boolean createDatabase() { + // 查询现有数据库 + QueryRequest showDatabases = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql("SHOW DATABASES;") + .build(); + Result result = null; + try { + CompletableFuture> future = greptimeDb.query(showDatabases); + result = future.get(); + } catch (Exception e) { + log.info("TABLE_NOT_EXIST: {}", e.getMessage()); + String msg = e.getMessage(); + if (msg != null && !msg.contains(DATABASE_NOT_EXIST)) { + log.warn(msg); + } - /** - * - * Checks if the database exists; if not, creates the Database. - * 检查数据库是否存在；如果不存在，则创建该数据库 - */ - private boolean createDatabase() { - // 查询现有数据库 - QueryRequest showDatabases = QueryRequest.newBuilder() - .exprType(SelectExprType.Sql) - .ql("SHOW DATABASES;") - .build(); - Result result = null; - try { - CompletableFuture> future = greptimeDb.query(showDatabases); - result = future.get(); - } catch (Exception e) { - log.info("TABLE_NOT_EXIST: {}",e.getMessage()); - String msg = e.getMessage(); - if (msg != null && !msg.contains(DATABASE_NOT_EXIST)) { - log.warn(msg); - } + } + // Check if the database exists; + // 检查现有数据库是否包括“hertzbeat” + boolean isDatabaseExist = false; + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + List rowsList = rows.collect(); + for (Row row : rowsList) { + for (io.greptime.models.Value value : row.values()) { + if (value.value().toString().equals(STORAGE_DATABASE)) { + log.info("Exist Database {}", STORAGE_DATABASE); + isDatabaseExist = true; + break; + } + } + } + } + // If it does not exist, create database + // 如果“hertzbeat”数据库不存在，则创建该数据库 + if (!isDatabaseExist) { + QueryRequest createDatabase = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql("CREATE DATABASE " + STORAGE_DATABASE + ";") + .build(); + try { + CompletableFuture> createFuture = greptimeDb.query(createDatabase); + isDatabaseExist = createFuture.get().isOk(); + log.info("Database {} does not exist,and has been created", STORAGE_DATABASE); + } catch (InterruptedException | ExecutionException e) { + log.error("Error creating database"); + } + } + return isDatabaseExist; + } - } - // Check if the database exists; - // 检查现有数据库是否包括“hertzbeat” - boolean isDatabaseExist = false; - if (result != null && result.isOk()) { - QueryOk queryOk = result.getOk(); - SelectRows rows = queryOk.getRows(); - List rowsList = rows.collect(); - for (Row row : rowsList) { - for (io.greptime.models.Value value : row.values()) { - if (value.value().toString().equals(STORAGE_DATABASE)) { - log.info("Exist Database {}",STORAGE_DATABASE); - isDatabaseExist = true; - break; - } - } - } - } - // If it does not exist, create database - // 如果“hertzbeat”数据库不存在，则创建该数据库 - if (!isDatabaseExist) { - QueryRequest createDatabase = QueryRequest.newBuilder() - .exprType(SelectExprType.Sql) - .ql("CREATE DATABASE " + STORAGE_DATABASE + ";") - .build(); - try { - CompletableFuture> createFuture = greptimeDb.query(createDatabase); - isDatabaseExist = createFuture.get().isOk(); - log.info("Database {} does not exist,and has been created",STORAGE_DATABASE); - } catch (InterruptedException | ExecutionException e) { - log.error("Error creating database"); - } - } - return isDatabaseExist; - } + @Override + void saveData(CollectRep.MetricsData metricsData) { + if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) { + return; + } + if (metricsData.getValuesList().isEmpty()) { + log.info("[warehouse greptime] flush metrics data {} is null, ignore.", metricsData.getId()); + return; + } + String monitorId = String.valueOf(metricsData.getId()); + String table = metricsData.getApp() + "_" + metricsData.getMetrics(); + //TODO bug：选择STORAGE_DATABASE不起作用，还是默认存在public里 + TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); - @Override - void saveData(CollectRep.MetricsData metricsData) { - if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) { - return; - } - if (metricsData.getValuesList().isEmpty()) { - log.info("[warehouse greptime] flush metrics data {} is null, ignore.", metricsData.getId()); - return; - } - String monitorId = String.valueOf(metricsData.getId()); - String table = metricsData.getApp() + "_" + metricsData.getMetrics(); - //TODO bug：选择STORAGE_DATABASE不起作用，还是默认存在public里 - TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); + List semanticTypes = new LinkedList<>(Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp)); + List dataTypes = new LinkedList<>(Arrays.asList(ColumnDataType.String, ColumnDataType.String, ColumnDataType.Int64)); + List columnNames = new LinkedList<>(Arrays.asList("monitor_id", "instance", "ts")); - List semanticTypes = new LinkedList<>(Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp)); - List dataTypes = new LinkedList<>(Arrays.asList(ColumnDataType.String, ColumnDataType.String, ColumnDataType.Int64)); - List columnNames = new LinkedList<>(Arrays.asList("monitor_id", "instance", "ts")); + List fieldsList = metricsData.getFieldsList(); + for (CollectRep.Field field : fieldsList) { + semanticTypes.add(SemanticType.Field); + columnNames.add(field.getName()); + // handle field type + if (field.getType() == CommonConstants.TYPE_NUMBER) { + dataTypes.add(ColumnDataType.Float64); + } else if (field.getType() == CommonConstants.TYPE_STRING) { + dataTypes.add(ColumnDataType.String); + } + } + tableSchemaBuilder.semanticTypes(semanticTypes.toArray(new SemanticType[0])); + tableSchemaBuilder.dataTypes(dataTypes.toArray(new ColumnDataType[0])); + tableSchemaBuilder.columnNames(columnNames.toArray(new String[0])); + WriteRows rows = WriteRows.newBuilder(tableSchemaBuilder.build()).build(); + try { + long now = System.currentTimeMillis(); + Object[] values = new Object[3 + fieldsList.size()]; + values[0] = monitorId; + values[2] = now; + for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + String instance = valueRow.getInstance(); + if (!instance.isEmpty()) { + instance = String.format("\"%s\"", instance); + values[1] = instance; + } else { + values[1] = null; + } + for (int i = 0; i < fieldsList.size(); i++) { + if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { + if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) { + values[3 + i] = Double.parseDouble(valueRow.getColumns(i)); + } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) { + values[3 + i] = valueRow.getColumns(i); + } + } else { + values[3 + i] = null; + } + } + rows.insert(values); + } + rows.finish(); + CompletableFuture> writeFuture = greptimeDb.write(rows); + writeFuture.whenComplete((result, throwable) -> { + if (throwable != null) { + log.error("[warehouse greptime]-write data error:{}", result, throwable); + } + }); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } - List fieldsList = metricsData.getFieldsList(); - for (CollectRep.Field field : fieldsList) { - semanticTypes.add(SemanticType.Field); - columnNames.add(field.getName()); - // handle field type - if (field.getType() == CommonConstants.TYPE_NUMBER) { - dataTypes.add(ColumnDataType.Float64); - } else if (field.getType() == CommonConstants.TYPE_STRING) { - dataTypes.add(ColumnDataType.String); - } - } - tableSchemaBuilder.semanticTypes(semanticTypes.toArray(new SemanticType[0])); - tableSchemaBuilder.dataTypes(dataTypes.toArray(new ColumnDataType[0])); - tableSchemaBuilder.columnNames(columnNames.toArray(new String[0])); - WriteRows rows = WriteRows.newBuilder(tableSchemaBuilder.build()).build(); - try { - long now = System.currentTimeMillis(); - Object[] values = new Object[3 + fieldsList.size()]; - values[0] = monitorId; - values[2] = now; - for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { - String instance = valueRow.getInstance(); - if (!instance.isEmpty()) { - instance = String.format("\"%s\"", instance); - values[1] = instance; - } else { - values[1] = null; - } - for (int i = 0; i < fieldsList.size(); i++) { - if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { - if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) { - values[3 + i] = Double.parseDouble(valueRow.getColumns(i)); - } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) { - values[3 + i] = valueRow.getColumns(i); - } - } else { - values[3 + i] = null; - } - } - rows.insert(values); - } - rows.finish(); - CompletableFuture> writeFuture = greptimeDb.write(rows); - writeFuture.whenComplete((result, throwable) -> { - if (throwable != null) { - log.error("[warehouse greptime]-write data error:{}", result, throwable); - } - }); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } + @Override + public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, + String instance, String history) { + Map> instanceValuesMap = new HashMap<>(8); + if (!isServerAvailable()) { + log.error("\n\t---------------GrepTime Init Failed---------------\n" + + "\t--------------Please Config GrepTime--------------\n" + + "\t----------Can Not Use Metric History Now----------\n"); + return instanceValuesMap; + } + long expireTime = getExpireTimeFromToken(history); + String table = app + "_" + metrics; + String selectSql = instance == null ? + String.format(QUERY_HISTORY_SQL, metric, table, expireTime, monitorId) + : String.format(QUERY_HISTORY_WITH_INSTANCE_SQL, metric, table, expireTime, monitorId, instance); + log.debug("selectSql: {}", selectSql); + QueryRequest request = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql(selectSql) + .build(); + try { + CompletableFuture> future = greptimeDb.query(request); + Result result = future.get(); + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + List> maps = rows.collectToMaps(); + List valueList; + for (Map map : maps) { + String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(strValue, (long) map.get("ts"))); + } + } + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && msg.contains(TABLE_NOT_EXIST)) { + List valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(null, System.currentTimeMillis())); + log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + return instanceValuesMap; + } - @Override - public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, - String instance, String history) { - Map> instanceValuesMap = new HashMap<>(8); - if (!isServerAvailable()) { - log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config GrepTime--------------\n" + - "\t----------Can Not Use Metric History Now----------\n"); - return instanceValuesMap; - } - long expireTime = 0; - try { - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - expireTime = dateTime.toEpochSecond() * 1000; - } catch (Exception e) { - log.error("parse history time error: {}. use default: 6h", e.getMessage()); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6)); - expireTime = dateTime.toEpochSecond() * 1000; - } - String table = app + "_" + metrics; - String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, expireTime,monitorId); - log.debug("selectSql: {}", selectSql); - QueryRequest request = QueryRequest.newBuilder() - .exprType(SelectExprType.Sql) - .ql(selectSql) - .build(); - try { - CompletableFuture> future = greptimeDb.query(request); - Result result = future.get(); - if (result != null && result.isOk()) { - QueryOk queryOk = result.getOk(); - SelectRows rows = queryOk.getRows(); - List> maps = rows.collectToMaps(); - List valueList; - for (Map map : maps) { - String strValue = new BigDecimal(map.get(metric).toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); - valueList.add(new Value(strValue, (long) map.get("ts"))); - } - } - } catch (FlightRuntimeException e) { - String msg = e.getMessage(); - if (msg != null && msg.contains(TABLE_NOT_EXIST)) { - List valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); - valueList.add(new Value(null, System.currentTimeMillis())); - log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } - return instanceValuesMap; - } + private long getExpireTimeFromToken(String history) { + long expireTime; + try { + TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); + ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); + expireTime = dateTime.toEpochSecond() * 1000; + } catch (Exception e) { + log.error("parse history time error: {}. use default: 6h", e.getMessage()); + ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6)); + expireTime = dateTime.toEpochSecond() * 1000; + } + return expireTime; + } - @Override - public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, - String metric, String instance, String history) { - Map> instanceValuesMap = new HashMap<>(8); - if (!isServerAvailable()) { - log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config GrepTime--------------\n" + - "\t----------Can Not Use Metric History Now----------\n"); - return instanceValuesMap; - } - String table = app + "_" + metrics; - List instances = new LinkedList<>(); - if (instance != null) { - instances.add(instance); - } - if (instances.isEmpty()){ - String selectSql = String.format(QUERY_INSTANCE_SQL, table); - log.debug("selectSql: {}", selectSql); - QueryRequest request = QueryRequest.newBuilder() - .exprType(SelectExprType.Sql) - .ql(selectSql) - .build(); - try { - CompletableFuture> future = greptimeDb.query(request); - Result result = future.get(); - if (result != null && result.isOk()) { - QueryOk queryOk = result.getOk(); - SelectRows rows = queryOk.getRows(); - while (rows.hasNext()){ - Row row = rows.next(); - if (row !=null){ - List values = row.values(); - for (io.greptime.models.Value value : values){ - log.info("value:{}", value.value()); - Object instanceValue = value.value(); - if (instanceValue == null || "".equals(instanceValue)) { - instances.add("''"); - } else { - instances.add(instanceValue.toString()); - } - } - } + @Override + public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, + String metric, String instance, String history) { + Map> instanceValuesMap = new HashMap<>(8); + if (!isServerAvailable()) { + log.error("\n\t---------------GrepTime Init Failed---------------\n" + + "\t--------------Please Config GrepTime--------------\n" + + "\t----------Can Not Use Metric History Now----------\n"); + return instanceValuesMap; + } + String table = app + "_" + metrics; + List instances = new LinkedList<>(); + if (instance != null) { + instances.add(instance); + } + if (instances.isEmpty()) { + String selectSql = String.format(QUERY_INSTANCE_SQL, table); + log.debug("selectSql: {}", selectSql); + QueryRequest request = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql(selectSql) + .build(); + try { + CompletableFuture> future = greptimeDb.query(request); + Result result = future.get(); + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + while (rows.hasNext()) { + Row row = rows.next(); + if (row != null) { + List values = row.values(); + for (io.greptime.models.Value value : values) { + log.debug("value:{}", value.value()); + Object instanceValue = value.value(); + if (instanceValue == null || "".equals(instanceValue)) { + instances.add("''"); + } else { + instances.add(instanceValue.toString()); + } + } + } - } - } - } catch (FlightRuntimeException e) { - String msg = e.getMessage(); - if (msg != null && msg.contains(TABLE_NOT_EXIST)) { - log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } - long startTime; - long endTime; - try { - TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history); - ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount); - startTime = dateTime.toEpochSecond() * 1000; - } catch (Exception e) { - log.error("parse history time error: {}. use default: 6h", e.getMessage()); - ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6)); - startTime = dateTime.toEpochSecond() * 1000; - } + } + } + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && msg.contains(TABLE_NOT_EXIST)) { + log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + long endTime; + long startTime = getExpireTimeFromToken(history); - Calendar cal = Calendar.getInstance(); + Calendar cal = Calendar.getInstance(); - long interval = System.currentTimeMillis() - startTime; - long fourHourCount = TimeUnit.MILLISECONDS.toHours(interval) / 4; - for (int i = 0; i < fourHourCount; i++){ - cal.clear(); - cal.setTimeInMillis(startTime); - cal.add(Calendar.HOUR_OF_DAY, 4); - endTime = cal.getTimeInMillis(); + long interval = System.currentTimeMillis() - startTime; + long fourHourCount = TimeUnit.MILLISECONDS.toHours(interval) / 4; + for (int i = 0; i < fourHourCount; i++) { + cal.clear(); + cal.setTimeInMillis(startTime); + cal.add(Calendar.HOUR_OF_DAY, 4); + endTime = cal.getTimeInMillis(); - for (String instanceValue:instances){ - String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, table, monitorId, startTime, endTime, metric, metric, metric, table, startTime, endTime); + for (String instanceValue : instances) { + String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, table, monitorId, startTime, endTime, metric, metric, metric, table, startTime, endTime); - log.info("selectSql: {}", selectSql); - QueryRequest request = QueryRequest.newBuilder() - .exprType(SelectExprType.Sql) - .ql(selectSql) - .build(); - List values = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); - try { - CompletableFuture> future = greptimeDb.query(request); - Result result = future.get(); - log.info("result:{}", result); - if (result != null && result.isOk()) { - QueryOk queryOk = result.getOk(); - SelectRows rows = queryOk.getRows(); - String[] col = new String[4]; - while (rows.hasNext()){ - Row row = rows.next(); - if (!row.values().isEmpty()){ - for (int j = 0; j < row.values().size(); j++){ - log.info("value:{}", row.values().get(j)); - String colStr = new BigDecimal(row.values().get(j).value().toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); - col[j] = colStr; - } - Value valueBuild = Value.builder() - .origin(col[0]).mean(col[1]) - .min(col[2]).max(col[3]) - .time(System.currentTimeMillis()) - .build(); - values.add(valueBuild); - } - } - log.info("values:{}", values); - } - } catch (FlightRuntimeException e) { - String msg = e.getMessage(); - if (msg != null && msg.contains(TABLE_NOT_EXIST)) { - List valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); - valueList.add(new Value(null, System.currentTimeMillis())); - log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); - } - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } - startTime = endTime; - } + log.debug("selectSql: {}", selectSql); + QueryRequest request = QueryRequest.newBuilder() + .exprType(SelectExprType.Sql) + .ql(selectSql) + .build(); + List values = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); + try { + CompletableFuture> future = greptimeDb.query(request); + Result result = future.get(); + log.debug("result:{}", result); + if (result != null && result.isOk()) { + QueryOk queryOk = result.getOk(); + SelectRows rows = queryOk.getRows(); + String[] col = new String[4]; + while (rows.hasNext()) { + Row row = rows.next(); + if (!row.values().isEmpty()) { + for (int j = 0; j < row.values().size(); j++) { + log.debug("value:{}", row.values().get(j)); + String colStr = new BigDecimal(row.values().get(j).value().toString()).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + col[j] = colStr; + } + Value valueBuild = Value.builder() + .origin(col[0]).mean(col[1]) + .min(col[2]).max(col[3]) + .time(System.currentTimeMillis()) + .build(); + values.add(valueBuild); + } + } + log.debug("values:{}", values); + } + } catch (FlightRuntimeException e) { + String msg = e.getMessage(); + if (msg != null && msg.contains(TABLE_NOT_EXIST)) { + List valueList = instanceValuesMap.computeIfAbsent(metric, k -> new LinkedList<>()); + valueList.add(new Value(null, System.currentTimeMillis())); + log.info("[warehouse greptime]-TABLE_NOT_EXIST: {}", table); + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + startTime = endTime; + } - return instanceValuesMap; - } + return instanceValuesMap; + } - @Override - public void destroy() { - if (this.greptimeDb != null) { - this.greptimeDb.shutdownGracefully(); - } - } + @Override + public void destroy() { + if (this.greptimeDb != null) { + this.greptimeDb.shutdownGracefully(); + } + } } From 50f333f7c5925980c7b2f20c2a445bf8fa8be961 Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Mon, 3 Apr 2023 00:14:23 +0800 Subject: [PATCH 21/28] [warehouse]Add support for greptimedb. --- .../hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index 9f863f5ac32..68e8ed4a185 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -277,6 +277,7 @@ private long getExpireTimeFromToken(String history) { } @Override + //TODO greptime未找到合适的sql函数处理，暂时使用代码实现，将来greptime更新文档改用sql实现 public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); From 6b7225e4c7985bb1d870c786314ea62957c07771 Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Mon, 3 Apr 2023 00:30:07 +0800 Subject: [PATCH 22/28] [warehouse]Add support for greptimedb. --- .../warehouse/store/HistoryGrepTimeDbDataStorage.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index 68e8ed4a185..815f44dc9a3 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -118,7 +118,7 @@ private boolean createDatabase() { List rowsList = rows.collect(); for (Row row : rowsList) { for (io.greptime.models.Value value : row.values()) { - if (value.value().toString().equals(STORAGE_DATABASE)) { + if (STORAGE_DATABASE.equals(value.value().toString())) { log.info("Exist Database {}", STORAGE_DATABASE); isDatabaseExist = true; break; @@ -277,7 +277,6 @@ private long getExpireTimeFromToken(String history) { } @Override - //TODO greptime未找到合适的sql函数处理，暂时使用代码实现，将来greptime更新文档改用sql实现 public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); @@ -331,6 +330,9 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str log.error(e.getMessage(), e); } } + /* + TODO greptime未找到合适的sql函数处理，暂时使用代码实现，将来greptime更新文档改用sql实现 + */ long endTime; long startTime = getExpireTimeFromToken(history); From 36a9a7c55dbcfe140f7f8ff0f50e0f893fdaf3ac Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Mon, 3 Apr 2023 20:33:25 +0800 Subject: [PATCH 23/28] Update warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java Co-authored-by: JeremyHi Signed-off-by: zqr10159 --- .../hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index 815f44dc9a3..a130b165650 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -282,7 +282,7 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config GrepTime--------------\n" + + "\t--------------Please Config Greptime--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } From 32201232258dab867fdc67f42e37c0047fd6116f Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Mon, 3 Apr 2023 20:33:41 +0800 Subject: [PATCH 24/28] Update warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java Co-authored-by: JeremyHi Signed-off-by: zqr10159 --- .../hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index a130b165650..8a515d08887 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -131,7 +131,7 @@ private boolean createDatabase() { if (!isDatabaseExist) { QueryRequest createDatabase = QueryRequest.newBuilder() .exprType(SelectExprType.Sql) - .ql("CREATE DATABASE " + STORAGE_DATABASE + ";") + .ql("CREATE DATABASE %s;", STORAGE_DATABASE) .build(); try { CompletableFuture> createFuture = greptimeDb.query(createDatabase); From 080c03232e06898b3f060ea2ff7226d38aea175a Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Mon, 3 Apr 2023 21:42:49 +0800 Subject: [PATCH 25/28] Update warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java Co-authored-by: JeremyHi Signed-off-by: tomsun28 --- .../hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index 8a515d08887..94a74ca5178 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -281,7 +281,7 @@ public Map> getHistoryIntervalMetricData(Long monitorId, Str String metric, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { - log.error("\n\t---------------GrepTime Init Failed---------------\n" + + log.error("\n\t---------------Greptime Init Failed---------------\n" + "\t--------------Please Config Greptime--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; From 65b5ec0e0cb4f2ee15a3d03e0d3093daba45abad Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Tue, 4 Apr 2023 00:01:04 +0800 Subject: [PATCH 26/28] [warehouse] 1.Update greptimedb `0.1.2` to `0.1.3`, 2.Use async method `future.get()` to write rows, 3.Update `GrepTime` to `Greptime` --- warehouse/pom.xml | 2 +- .../store/HistoryGrepTimeDbDataStorage.java | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 746c2dd2142..587a882bc89 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -28,7 +28,7 @@ 0.13.3 3.0.5 3.0.0 - 0.1.2 + 0.1.3 4.0.0 diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index 94a74ca5178..1325989b856 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -81,7 +81,7 @@ private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProper .build(); greptimeDb = new GreptimeDB(); if (!greptimeDb.init(opts)) { - log.error("Fail to start GreptimeDB client"); + log.error("Fail to start Greptime client"); return false; } return createDatabase(); @@ -155,7 +155,6 @@ void saveData(CollectRep.MetricsData metricsData) { } String monitorId = String.valueOf(metricsData.getId()); String table = metricsData.getApp() + "_" + metricsData.getMetrics(); - //TODO bug：选择STORAGE_DATABASE不起作用，还是默认存在public里 TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); List semanticTypes = new LinkedList<>(Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp)); @@ -203,13 +202,19 @@ void saveData(CollectRep.MetricsData metricsData) { } rows.insert(values); } + rows.finish(); CompletableFuture> writeFuture = greptimeDb.write(rows); - writeFuture.whenComplete((result, throwable) -> { - if (throwable != null) { - log.error("[warehouse greptime]-write data error:{}", result, throwable); + try { + Result result = writeFuture.get(10, TimeUnit.SECONDS); + if (result.isOk()) { + System.out.println("[warehouse greptime]-Write successful"); + } else { + System.out.println("[warehouse greptime]-Write failed: " + result.getErr().getFailedQl()); } - }); + } catch (Throwable throwable) { + System.err.println("[warehouse greptime]-Error occurred: " + throwable.getMessage()); + } } catch (Exception e) { log.error(e.getMessage(), e); } @@ -220,8 +225,8 @@ public Map> getHistoryMetricData(Long monitorId, String app, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { - log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config GrepTime--------------\n" + + log.error("\n\t---------------Greptime Init Failed---------------\n" + + "\t--------------Please Config Greptime--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } From f965433f53ce6f33fd8fcda9cee2d4b1dd808047 Mon Sep 17 00:00:00 2001 From: zqr10159 Date: Tue, 4 Apr 2023 00:04:50 +0800 Subject: [PATCH 27/28] [warehouse] 1.Update greptimedb `0.1.2` to `0.1.3`, 2.Use sync method `future.get()` to write rows, 3.Update `GrepTime` to `Greptime` --- warehouse/pom.xml | 2 +- .../store/HistoryGrepTimeDbDataStorage.java | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 746c2dd2142..587a882bc89 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -28,7 +28,7 @@ 0.13.3 3.0.5 3.0.0 - 0.1.2 + 0.1.3 4.0.0 diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java index 94a74ca5178..1325989b856 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryGrepTimeDbDataStorage.java @@ -81,7 +81,7 @@ private boolean initDbSession(WarehouseProperties.StoreProperties.GreptimeProper .build(); greptimeDb = new GreptimeDB(); if (!greptimeDb.init(opts)) { - log.error("Fail to start GreptimeDB client"); + log.error("Fail to start Greptime client"); return false; } return createDatabase(); @@ -155,7 +155,6 @@ void saveData(CollectRep.MetricsData metricsData) { } String monitorId = String.valueOf(metricsData.getId()); String table = metricsData.getApp() + "_" + metricsData.getMetrics(); - //TODO bug：选择STORAGE_DATABASE不起作用，还是默认存在public里 TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(TableName.with(STORAGE_DATABASE, table)); List semanticTypes = new LinkedList<>(Arrays.asList(SemanticType.Tag, SemanticType.Tag, SemanticType.Timestamp)); @@ -203,13 +202,19 @@ void saveData(CollectRep.MetricsData metricsData) { } rows.insert(values); } + rows.finish(); CompletableFuture> writeFuture = greptimeDb.write(rows); - writeFuture.whenComplete((result, throwable) -> { - if (throwable != null) { - log.error("[warehouse greptime]-write data error:{}", result, throwable); + try { + Result result = writeFuture.get(10, TimeUnit.SECONDS); + if (result.isOk()) { + System.out.println("[warehouse greptime]-Write successful"); + } else { + System.out.println("[warehouse greptime]-Write failed: " + result.getErr().getFailedQl()); } - }); + } catch (Throwable throwable) { + System.err.println("[warehouse greptime]-Error occurred: " + throwable.getMessage()); + } } catch (Exception e) { log.error(e.getMessage(), e); } @@ -220,8 +225,8 @@ public Map> getHistoryMetricData(Long monitorId, String app, String instance, String history) { Map> instanceValuesMap = new HashMap<>(8); if (!isServerAvailable()) { - log.error("\n\t---------------GrepTime Init Failed---------------\n" + - "\t--------------Please Config GrepTime--------------\n" + + log.error("\n\t---------------Greptime Init Failed---------------\n" + + "\t--------------Please Config Greptime--------------\n" + "\t----------Can Not Use Metric History Now----------\n"); return instanceValuesMap; } From 051b90496b84ff780bde2aaf8ccb6a8dd4d6929a Mon Sep 17 00:00:00 2001 From: tomsun28 Date: Tue, 4 Apr 2023 09:04:59 +0800 Subject: [PATCH 28/28] Update warehouse/pom.xml Signed-off-by: tomsun28 --- warehouse/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 587a882bc89..28b20f59cc6 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -79,12 +79,6 @@ iotdb-session ${iotdb-session.version} - - - - - - io.greptime greptimedb-protocol