From e540620ad980425942fef3435bf2e9de6d0940cd Mon Sep 17 00:00:00 2001 From: yaojin Date: Wed, 6 Dec 2023 18:12:19 +0800 Subject: [PATCH] fixbug 0.2.0 --- docker/docker-compose.yml | 8 + docker/office/bisheng/all.js | 487 ++++++++++++++++++ docker/office/bisheng/bisheng.js | 15 + docker/office/bisheng/config.json | 35 ++ docker/office/bisheng/icon.png | Bin 0 -> 17709 bytes docker/office/bisheng/index.html | 12 + src/backend/bisheng/api/v1/knowledge.py | 13 +- src/backend/bisheng/cache/utils.py | 7 +- .../bisheng/database/models/message.py | 4 +- .../bisheng/interface/chains/custom.py | 4 +- src/backend/bisheng/main.py | 1 - .../bisheng/template/frontend_node/chains.py | 9 +- src/backend/bisheng/utils/minio_client.py | 3 - .../chat_models/proxy_llm.py | 2 + .../bisheng_langchain/vectorstores/milvus.py | 0 15 files changed, 584 insertions(+), 16 deletions(-) create mode 100644 docker/office/bisheng/all.js create mode 100644 docker/office/bisheng/bisheng.js create mode 100644 docker/office/bisheng/config.json create mode 100644 docker/office/bisheng/icon.png create mode 100644 docker/office/bisheng/index.html create mode 100644 src/bisheng-langchain/bisheng_langchain/vectorstores/milvus.py diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 937d2eb11..172189084 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,6 +23,13 @@ services: - ${DOCKER_VOLUME_DIRECTORY:-.}/mysql/conf/my.cnf:/etc/mysql/my.cnf - ${DOCKER_VOLUME_DIRECTORY:-.}/mysql/data:/var/lib/mysql + office: + image: onlyoffice/documentserver:7.1.1 + ports: + - "8701:80" + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/office/bisheng:/var/www/onlyoffice/documentserver/sdkjs-plugins/bisheng + backend: image: dataelement/bisheng-backend:latest healthcheck: @@ -42,6 +49,7 @@ services: restart: on-failure depends_on: - "mysql" + - "office" - "redis" nginx: diff --git a/docker/office/bisheng/all.js b/docker/office/bisheng/all.js new file mode 100644 index 000000000..93560514a --- /dev/null +++ b/docker/office/bisheng/all.js @@ -0,0 +1,487 @@ +(function (window, undefined) { + let selectText = '' + + window.Asc.plugin.init = function (e) { + selectText = e + } + window.Asc.plugin.event_onClick = function () { + selectText = '' + } + + window.Asc.plugin.button = function (id) { + } + + const EventMap = { + sendToParent (method, data) { + let params = { + type: 'onExternalFrameMessage', + method, + data + } + window.top.postMessage(JSON.stringify(params), location.origin) + }, + focusInDocument (data) { + window.Asc.scope.field = { + id: data.id, + fieldFlag: data.fieldFlag, + $index: data.$index || 1 + } + window.Asc.plugin.callCommand(function () { + let field = Asc.scope.field || {} + let index = field.$index ? field.$index : 1 + let oDoc = Api.GetDocument() + let flag = `{{${field.fieldFlag}}}` + let oRange = oDoc.Search(flag) + let cur = 1 + for (let i = 0; i < oRange.length; i++) { + if (oRange[i].GetText() === flag) { + if (cur === index) { + oRange[i].Select() + break + } + cur = cur + 1 + } + } + }) + }, + focusTableInDoc (data) { + window.Asc.scope.marker = data.marker + window.Asc.plugin.callCommand(function () { + let flag = Asc.scope.marker || '' + let oDoc = Api.GetDocument() + let oRange = oDoc.GetBookmarkRange(flag) + oRange.Select() + }) + }, + addMarker (data) { + let flag = '{{' + data.fieldFlag + '}}' + window.Asc.plugin.executeMethod('PasteText', [flag]) + }, + addBookMarker (data) { + window.Asc.scope.value = data + window.Asc.plugin.callCommand(function () { + let oDoc = Api.GetDocument() + let range = oDoc.GetRangeBySelect() + let params = { + type: 'onExternalFrameMessage', + method: 'addBookMarker' + } + let marker = Asc.scope.value + let markers = [] + if (range) { + let texts = range.GetText() + let pars = range.GetAllParagraphs() || [] + let txtList = [] + for (let i = 0; i < pars.length; i++) { + let text = pars[i].GetText() + txtList.push(text) + } + let table = pars[0] ? pars[0].GetParentTable() : null + let count = table ? table.GetRowsCount() : 0 + for (let i = 0; i < count; i++) { + let row = table.GetRow(i) + let firstCell = row.GetCell(0) + let cellText = firstCell ? firstCell.GetContent().GetElement(0).GetText() : '' + // 序号 + let isNumbering = false + if (firstCell.GetContent().GetElement(0).GetNumbering()) { + isNumbering = true + let cellCount = row.GetCellsCount() + for (let j = 1; j < cellCount; j++) { + let cellItem = row.GetCell(j) + if (!cellItem.GetContent().GetElement(0).GetNumbering()) { + cellText = cellItem.GetContent().GetElement(0).GetText() + firstCell = cellItem + break + } + } + } + if (cellText && txtList.includes(cellText)) { + let cRange = firstCell.Search(cellText)[0] + cRange.AddBookmark(marker.key + i) + markers.push(marker.key + i) + } + } + // range.AddBookmark(Asc.scope.value.key) + params.data = Object.assign(marker, { + key: markers.join(','), + texts + }) + } else { + params.data = false + } + window.top.postMessage(JSON.stringify(params), location.origin) + }) + }, + deleteBookMarker (data) { + window.Asc.scope.value = data + window.Asc.plugin.callCommand(function () { + let oDoc = Api.GetDocument() + let markers = Asc.scope.value || [] + for (let i = 0; i < markers.length; i++) { + oDoc.DeleteBookmark(markers[i]) + } + }) + }, + // 批量删除循环应用内标签 + deleteLoopApp (list) { + window.Asc.scope.value = list + window.Asc.plugin.callCommand(function () { + let list = window.Asc.scope.value || [] + let oDoc = Api.GetDocument() + list.forEach(row => { + if (row.loopType === 0) { + oDoc.SearchAndReplace({ searchString: `{{${row.startTag}}}`, replaceString: '' }, `{{${row.startTag}}}`, '') + oDoc.SearchAndReplace({ searchString: `{{${row.endTag}}}`, replaceString: '' }, `{{${row.endTag}}}`, '') + } else if (row.loopType === 1) { + oDoc.DeleteBookmark(row.bookmark) + } + }) + }) + }, + // 更新占位符 + replaceMarker (data) { + window.Asc.scope.st = '{{' + data.newValue + '}}' + // 原来的值 + if (data.oldValue) { + window.Asc.scope.old = '{{' + data.oldValue + '}}' + } else { + this.addMarker(data) + return + } + window.Asc.plugin.callCommand(function () { + let oDocument = Api.GetDocument() + oDocument.SearchAndReplace({ searchString: Asc.scope.old, replaceString: Asc.scope.st }, Asc.scope.old, Asc.scope.st) + }, false) + }, + // 查找并插入占位符 + findAndInsertMarker (data) { + window.Asc.scope.st = '{{' + data.fieldName + '}}' + window.Asc.scope.searchStr = data.fieldValue + window.Asc.plugin.callCommand(function () { + let oDocument = Api.GetDocument() + oDocument.SearchAndReplace({ searchString: Asc.scope.searchStr, replaceString: Asc.scope.st }, Asc.scope.searchStr, Asc.scope.st) + }, false) + }, + insertPosition (data) { + if (!selectText) { + let postData = { + text: selectText, + ...data, + selected: false + } + this.sendToParent('addRange', postData) + return false + } + window.Asc.scope.postData = data + window.Asc.plugin.callCommand(function() { + let postData = Asc.scope.postData || {} + let oDoc = Api.GetDocument() + let oRange = oDoc.GetRangeBySelect() + let selectText = oRange.GetText() + let oAllPar = oRange.GetAllParagraphs() + let oPar = oAllPar[oAllPar.length - 1] + let parText = oPar.GetText() + if (oAllPar.length > 1) { + oRange.AddText(`{{${postData.start}}}`, 'before') + if (selectText.includes(parText)) { + let newRange = oPar.GetRange(0, parText.length - 1) + newRange.AddText(`{{${postData.end}}}`, 'after') + } else { + oRange.AddText(`{{${postData.end}}}`, 'after') + } + } else { + let isEnd = parText.substr(0 - selectText.length) === selectText + isEnd = isEnd || selectText.includes(parText) + console.log('end = ', isEnd) + let start = Math.max(parText.indexOf(selectText), 0) + let end = start + Math.min(parText.length, selectText.length) - 1 + let newRange = oPar.GetRange(start, end) + oRange.AddText(`{{${postData.start}}}`, 'before') + newRange.AddText(`{{${postData.end}}}`, 'after') + } + + postData.selected = true + postData.text = selectText + let params = { + type: 'onExternalFrameMessage', + method: 'addRange', + data: postData + } + window.top.postMessage(JSON.stringify(params), location.origin) + }) + }, + deletePosition (data) { + window.Asc.scope.range = data + window.Asc.plugin.callCommand(function () { + let oDocument = Api.GetDocument() + let { start, end } = Asc.scope.range + let markers = [`{{${start}}}`, `{{${end}}}`] + for (let j = 0; j < markers.length; j++) { + oDocument.SearchAndReplace({ searchString: markers[j], replaceString: '' }, markers[j], '') + } + }) + }, + deletePositionMarker (data) { + window.Asc.scope.data = data + window.Asc.plugin.callCommand(function () { + let oDocument = Api.GetDocument() + let markers = Asc.scope.data || [] + for (let j = 0; j < markers.length; j++) { + oDocument.SearchAndReplace({ searchString: markers[j], replaceString: '' }, markers[j], '') + } + }) + }, + deletePositionArray (data) { + window.Asc.scope.data = data + window.Asc.plugin.callCommand(function () { + let oDocument = Api.GetDocument() + let markers = Asc.scope.data || [] + for (let j = 0; j < markers.length; j++) { + oDocument.SearchAndReplace({ searchString: markers[j], replaceString: '' }, markers[j], '') + } + }) + }, + replaceRangePosition (data) { + window.Asc.scope.list = data + window.Asc.plugin.callCommand(function () { + let list = Asc.scope.list || [] + let oDocument = Api.GetDocument() + list.forEach(row => { + oDocument.SearchAndReplace({ searchString: row.str, replaceString: row.newStr }, row.str, row.newStr) + }) + }) + }, + delMarker (data) { + let fields = [] + data.forEach(item => { + fields.push({ + text: '{{' + item.fieldFlag + '}}', + type: 'field' + }) + }) + window.Asc.scope.st = fields + window.Asc.plugin.callCommand(function () { + let oDocument = Api.GetDocument() + let markers = Asc.scope.st.slice(0) + for (let j = 0; j < markers.length; j++) { + let marker = markers[j] + if (marker.type === 'field') { + oDocument.SearchAndReplace({ searchString: marker.text, replaceString: '' }) + } + } + }) + }, + delMarkerGroup (data) { + this.delMarker(data.fields) + }, + // excel + insertCellName (field) { + window.Asc.scope.field = field + window.Asc.plugin.callCommand(function () { + let fieldItem = Asc.scope.field + let sheetObj = Api.GetActiveSheet() + let sheetName = sheetObj.GetName() + let oRange = Api.GetSelection() + let oCount = oRange.GetCount() + let params = { + type: 'onExternalFrameMessage', + method: 'addCellName' + } + if (oCount !== 1) { + params.data = false + } else { + let oAddr = oRange.GetAddress(true, true, '', false) + let sheetFlag = `${sheetName}!${oAddr}` + // let name = [fieldItem.fieldName, 'DEF', fieldItem.id].join('') + // let nameObj = sheetObj.GetDefName(name) + // console.log('inser cell before = ', name, sheetFlag, nameObj) + // let result = sheetObj.AddDefName(name, sheetFlag) + // console.log('insert cell ', name, sheetFlag, result) + fieldItem.fieldFlag = sheetFlag + fieldItem.$success = oAddr !== '' + params.data = fieldItem + } + window.top.postMessage(JSON.stringify(params), location.origin) + }) + }, + getFocusedCell () { + window.Asc.plugin.callCommand(function () { + let sheetObj = Api.GetActiveSheet() + let sheetName = sheetObj.GetName() + let oRange = Api.GetSelection() + let params = { + type: 'onExternalFrameMessage', + method: 'getFocusedCell' + } + let oAddr = oRange.GetAddress(true, true, '', false) + let sheetFlag = `${sheetName}!${oAddr}` + params.data = sheetFlag + window.top.postMessage(JSON.stringify(params), location.origin) + }) + }, + loadFileFlags (data) { + window.Asc.scope.list = data + window.Asc.plugin.callCommand(function () { + let list = Asc.scope.list || [] + let oDocument = Api.GetDocument() + let oParCount = oDocument.GetElementsCount() + let dataMap = {} + for (let i = 0; i < oParCount; i++) { + let oPar = oDocument.GetElement(i) + let ctype = oPar.GetClassType() + if (ctype === 'table') { + list.forEach(row => { + let flag = `{{${row.fieldFlag}}}` + let rs = oPar.Search(flag) + for (let i = 0; i < rs.length; i++) { + let oRange = rs[i] + if (oRange && oRange.GetText() === flag) { + if (dataMap[row.id]) { + dataMap[row.id] = dataMap[row.id] + 1 + } else { + dataMap[row.id] = 1 + } + } + } + }) + } else if (ctype === 'paragraph') { + let oParText = oPar.GetText() + list.forEach(row => { + let count = oParText.split(`{{${row.fieldFlag}}}`).length - 1 + if (dataMap[row.id]) { + dataMap[row.id] = dataMap[row.id] + count + } else { + dataMap[row.id] = count + } + }) + } + } + let params = { + type: 'onExternalFrameMessage', + method: 'loadFieldFlagCount', + data: dataMap + } + window.top.postMessage(JSON.stringify(params), location.origin) + }) + }, + /** + * data.sheetName: 要聚焦的sheet名称 + * data.cellName: 要聚焦的cell名称，如C1, D3等 + */ + focusCell (data) { + window.Asc.scope.data = data + window.Asc.plugin.callCommand(function () { + const theData = Asc.scope.data + const theSheet = Api.GetSheet(theData.sheetName || '') + if (theSheet) { + theSheet.SetActive() + + const theCell = theSheet.GetRange(theData.cellName || '') + if (theCell) { + theCell.Select() + } + } + }) + }, + + getSelectedText (data) { + window.Asc.scope.data = data + window.Asc.plugin.callCommand(function () { + const theData = Asc.scope.data + const oDoc = Api.GetDocument() + const oRange = oDoc.GetRangeBySelect() + if (oRange) { + const oParas = oRange.GetAllParagraphs() + // 只能选择一个段落，否则认为不成功 + if (oParas.length === 1) { + + const params = { + type: 'onExternalFrameMessage', + method: 'getSelectedText', + data: { + id: theData.id, + text: oRange.GetText() + } + } + window.top.postMessage(JSON.stringify(params), location.origin) + } + } + }) + } + } + + function receiveMessage (e) { + let data = e.data ? JSON.parse(e.data) : {} + if (data.type === 'onExternalPluginMessage') { + switch (data.method) { + case 'focus': + EventMap.focusInDocument(data.data) + break + case 'focusTable': + EventMap.focusTableInDoc(data.data) + break + case 'insert': + EventMap.addMarker(data.data) + break + case 'addBookMarker': + EventMap.addBookMarker(data.data) + break + case 'delBookMarker': + EventMap.deleteBookMarker(data.data) + break + case 'delLoopApp': + EventMap.deleteLoopApp(data.data) + break + case 'update': + EventMap.replaceMarker(data.data) + break + case 'findAndInsertMarker': + EventMap.findAndInsertMarker(data.data) + break + case 'addRange': + EventMap.insertPosition(data.data) + break + case 'updateRange': + EventMap.replaceRangePosition(data.data) + break + case 'delRange': + EventMap.deletePosition(data.data) + break + case 'delRangeArray': + EventMap.deletePositionArray(data.data) + break + case 'delQuoteGroup': + EventMap.deletePositionMarker(data.data) + break + case 'remove': + EventMap.delMarker([ data.data ]) + break + case 'removeQuestion': + EventMap.delMarkerGroup(data.data) + break + // excel + case 'addCellName': + EventMap.insertCellName(data.data) + break + case 'loadFieldFlagCount': + EventMap.loadFileFlags(data.data) + break + // 聚焦到某个单元格 + case 'focusCell': + EventMap.focusCell(data.data) + break + // 获取当前选中的单元格 + case 'getFocusedCell': + EventMap.getFocusedCell() + break + // 获取当前选中的文字 + case 'getSelectedText': + EventMap.getSelectedText(data.data) + break + } + } + } + + window.addEventListener('message', receiveMessage, false) +})(window, undefined) diff --git a/docker/office/bisheng/bisheng.js b/docker/office/bisheng/bisheng.js new file mode 100644 index 000000000..03507467e --- /dev/null +++ b/docker/office/bisheng/bisheng.js @@ -0,0 +1,15 @@ +(function () { + window.Asc.plugin.init = function (e) {} + window.Asc.plugin.event_onClick = function () {} + window.Asc.plugin.button = function (id) {} + + function onMessage(e) { + var data = e.data ? JSON.parse(e.data) : {} + if (data.action === 'insetMarker') { + const flag = '{{' + data.data + '}}' + window.Asc.plugin.executeMethod('PasteText', [flag]) + } + } + + window.addEventListener('message', onMessage, false) +})() \ No newline at end of file diff --git a/docker/office/bisheng/config.json b/docker/office/bisheng/config.json new file mode 100644 index 000000000..b6f13abbd --- /dev/null +++ b/docker/office/bisheng/config.json @@ -0,0 +1,35 @@ +{ + "name": "文档自动化", + "guid": "asc.{D2A0F3BE-CC8D-4956-BCD9-6CBEA6E8960E}", + "variations": [ + { + "description": "插入label-配置", + "url": "index.html", + "icons": [ + "icon.png", + "icon.png", + "icon.png", + "icon.png" + ], + "EditorsSupport": [ + "word", + "cell", + "slide" + ], + "isViewer": false, + "isVisual": false, + "isModal": true, + "isInsideMode": false, + "isSystem": false, + "initOnSelectionChanged": true, + "hideClose": true, + "initDataType": "text", + "isDisplayedInViewer": true, + "isUpdateOleOnResize": true, + "events": [ + "onClick", + "onTargetPositionChanged" + ] + } + ] +} \ No newline at end of file diff --git a/docker/office/bisheng/icon.png b/docker/office/bisheng/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..df532d700ea5e9ac052053b0a763315234076538 GIT binary patch literal 17709 zcmeHv2UJsCw`~+fET~i|D$-35q$4PxfRuz#lF*yd6MB~-d@3NlhEN586hf5_f*?vS z(gH{TL8^d&AfWUY@4f#A?+tzLj&aBP-?-!bJLBYR_L*7PdveZN>zp;=;N!sr;38OA zO&M_P7yxjL@&X)89{Z@KsA#4I(^OVdzyI$CS^<=HLKpyWaCCQtsVH32H!!^RYwREG zp@p0CKl;DfQ|xXI{<01L^b7vx`2T6>w564s1qJktvhlc5zML{yMoOR2=HL6*fA!7( zy?^Id-^<gek-qtnzJ;^vFMI?AbH~Bn)^5GcIO90?PAOKL&eK>Z= ziSt9(hZOr1k}2D9Yij^tjQ{}98UO$cJpjNNqklU_dHciMu2BwMrp(KU^0EQg1FQho z0BQh7fCWH^(h~&;14ICl2O|JQ%CGak)_xiJuUVjc`rs{qj^Qj{QO$KXHue`XVv3Z{T&0}W$0^`nXZL?J>Qg6aj-NPvjOq;KxC|WtaPm0d z^jWHtzZUfia_l(OiIdc)=;#?PvFIlBGcqxABJ|?o(S0NxOfL-=w}|#Vo`<3bWWYJf z9FEgb(E;QE+kjK+5AKZr<17Aq?@w*3#Y;ENLr2<^<@~ln;<=R2Ap7}#d@fBQ2vK$q zk}PUYLUYonBg&Oet8=rwJLO^%UM-^Sz`G=Wr6_OK@nUk^mC`bwy?*4{5Did_r#qG~ z1VS+;HK-XfW1Hy$AF{mTlX+|!59S*{l=SiaZg}n(mF@l@raW+-NnK8Cp0v#l;9|S@25VVi`8H73T zJqp#}yIsgeZ$nbf;u2eEiFh1KZLa{vDJHNAi1C1wybO3*nZ?wAxJ#SAN(C@ki03MH4YKd&3=}Q_t{{ zQKXU7wdA#D{{sIf*x;a5VOi6N`tVq}AlmC)qNaTi)O9->xv|LSFq0_)o7HrkeF9R# zastE9)!|@RlCnqyCr#sY_-m4jnxdC%ZFzwbnv*FPg~qW?^5B4R);QJ;Iy%7 zlsTW{_z964Pi9WBlPSt^Z-V*ICzpflUf=c!T(CfP7=F}^8FoO)ap>ULXj$uOnf^sHlLyZQ8??)C~Uk1@5qii_O1{fOY zevKD3e!k4o^zCNfo9n}}XJ^uO`p$T&aDOnLzC$c@n;EVvL``4LPWwLI(E5FNoGe;d zB<`+T-R&K9uWQY)Mv!0p23~L=Eb~pKJL{u>^{>fuWmwM>5zfvXM4ilR5 z1$vAyX7@%#C7I=&a%R$pObxuY`%X@lFhakENW#ey!8?9e;lOC*BBH(%+@JC5^R9}} zsbHl_;$k9_U<(Z(jM+j}^pk0tmD7{fGBt?39#lD8bO@F*V!xFr5xTR>+7EI%p#+ieReK0jevb~HKWV#`OUn(t!&G7g7%u?WM#k91R#wqrhRXCY&RjTa1L`-XLLk61&=? zcP|?{6C+H^Tm;38UfV9co>g@v`{DR3lh>?;5W^^N4~|bb!oReEAeP)BHtaoGZsV40 z&^fC;GbdW_X{viCzi2EWn!{A29@y5Vp{${-W;s3C*4DJ+>f&KJ*%W?B_vT^_;MB2y z%bfq$gy2ET;3XDu2YHCLC|sp?SZf1Rep^=1#i~EG-F6%%Ss>{v8A~*6k3)k^9D?qJ z0UbZxqw_%6={Q_+)7!2ViP&Le*a&LdW7p~z4e!@!P&MK{rik?w$Umwl0;K-3)%Q58nLk#g_h^ zvl@2j>5aAgO}8z#zW|a-x-b9N#s1@Gsy&_$X`kE)w6mSDa!QRFv2x%6kb81p!pV+Q zkU3XEN3tLi+*1ZRqmX%$LOmx#RcZT;B!XmIdq%9(K)B==^=ewyu`d6CvQXPV`eav2 zRT51C@U;3hbS4e>A&tVtKu)S4sw;A_YPs>qT-f{omnF%8@uxPPK{?+*c~}+s?p_|9 z6`PAiqyFk7=Ww*Bv7&ZU0-s0`-HoZbMZS`oUlLMbuEiF2D2H1}w?{kk+F5fD)z#eo zvk*gb%Jd#Nvh|cTbJ*h4bhE+`N$+=mIZojN!036PPM(fH!YL2b7&ucuw?371CD%hQ z3dM9PKAVP5?8O@|QMW_}b2OeVSq3%4qHr<7zvs#BHau!JjsdL$>w+N z0z@rxWrAbg{K)#$e*l2Lblm9aS+7r-^{-U5+tb4n4*=3(VY4^CeJQtw?%NrgBMb&)hF^P=DMex`)@rp;`eaZJ}Mmwy`KhKbVAb zovUy&OXWf_C)@`qH3BJlF6I1pKFS@wWnLv%S2Vwk`L1V27Y>rj{a)dvRf&F0T3|-^ zc8ygNAOiy^SDvn1(xRpTG-gZ2KUvZbFF5?lmFxf`2rgWaj%$A*QULc#o)iKVJMdy7 z9Eu?coTo8CwdFEbvu4!PuO%tFk%Q>^Th2yM?goJIb6rVQdn8oIcgaRD#wyIoYWvJ^UNg$4wENeXKGd( zig4Mlq^_>veHHIYM83pt8<@q?y{r^Oi`ZdmsD?Ydrk+`yGn^@WyXiXib7H8k_al3> zH(WOJC0lb3{PB&V){q_}%n@@X!Gf+s4}7H^cb)C#ZNJjC)DqKsdT1?dfpr+YX(e01 zjw9xV6`8B;Udg6a`rndRFY~0H`h@7VwWTgDV;qMFtZ|7m&m1v#WVvkHhPSC)RzNmo z;#B(NGfudKEjqTdu1CyCYBd!ae!XBLI37j&Sn$e|svikrWVU1Z4kxAh+IK@D|HK}3 z9VY7SsBe4n5-ng*@@eY4mA{hm)6t3}=8iD)H`BrY&533Th^M$=yOUmJ4iB4(dVPHQ zMmN>dr28dgxjvYjQb&xpB`{THqhuu;M86OT2tGdAAq3%-9RlB(^gJ|WwKkbF6!!v- zug^y0GWuj2L$)s0Hs-iOpZAQk!Y)H1s-Aze?8=`(dg~AP zf)Ji4khzj~sEgKYlSJxGcIAjgFeF4V^3I@3DU4A$9^8i?HRZM3y>@)+;@jW(1^|Fb zXFZ~d?4)CE#i0m)AsL)qzMY8ZE59OhR{bho15eU%FuR|p9=|ViBSR|x7D3XDtaFeSC3K+YYb|SHx%}Q-vDwII#}{B*GqS(xDBELs&-#2(7K0Z`c(rrIYto6LRqNo04fq1%LeTx0 zjvUeask6~spqba-i%pQzuY)n_x35T}Yxg^c_0384vE zo?6Rv@V;}Y3`KL^6FdRXQL(#P-ANUwf`f8yb z1}`@_K5pQ6%7!YgxexJ@USa%tI`jy_3Eysd{1qi{md%k6e(&|5Qf?x;fmT9)EDKkY z_9M*mMB~aKLh6tkwxowOuZ%LBRVIQ{@*R6&V;_8*ZXf>xd$j7pV;=>(tGK@@LUpBV ziH>u!lP{i#+&<)&_6UuCGtK$mo9H%!{Jyd!E}~f>i4A632KH?)H_h30WiKJXkxR3; z{kBH8VxRIkD5GA`um1DyMUe$h{W_>2}3fuNhs}|MRPv!#%!zm zwx9KDf@qGtoqSe>SY{a-AtborzIt7qzuU&N=bRBVnMQHCT9AJvEMk}@tZr4k{EajA zov^yKknomD;U3=spy;bnmOONqqhx=;S-#=`aG_;O{#5`f;OBV4<^doeW4mZ0Bf4YX zEd1yA#?KUWZ>i@_Bn%Vzoym~X$k4eCSg5^jE+dN;GQSx)&#xX+6^*b4dpzf9HU43U zz|f^BS(_IGmVGewn5+2ot^tRwF^|{E=}*ej$h{XW`(wIbATtY1HdV4VgoGkXq7vfm zbk$y0!b%O-T(u&CMoYtueFI+O$F}nw4VnsN>&M`Zd%AW#z%KJwW1pZcb1gq7ZL-7J zt%-G~Zaw4lx-mI)b$ppv>7U1raj}bb`hgg!*GXBYF4lKk;hXwCwd>s56j_QGr%JmQ zQ*(%j?6_6lQn{1sFrM4xL#3ZDN(I0m+pC z>o>>pb=hm$s&;=K04|HWSlACpy`8&gDRc<2aQz_Yvb4XD>a|d6`q!WU?b!6Fzh6KU z+pXdXRO*CVi0Gw5L~*qZa}|Yj4`|JXLD5}1rr_l@M|nB<7TU}DBNEVW(v7TVk3jT~ z&@=Q>BJ91Fw-iSP$5e3C4Tlu`&&!kN{zxEO)5DtD)adT$)%PmQl3j4W^Dfj1%ZC(8 z9Wp8Lh4ycom9J&t=8C-h2jn8j~aGrk(!D2kMM4rYM1E!-%j+ZZD3vQ`asrO}R(~pKss<*CUV3)(Ng7OQx2|OJ+ ztsOajci2C+b8}Y^KSo4E)0?Pm900-~*ev;Pt*ssVERp9L-Wxp&F>KCIC21NhM6wg( zmOUee5VP?H5_z~V+`SD+f<~%Dd(Ob@Y3EONDFL&xP7QD`ALQ$FBBNFCD;w#BS8{C z8ZbIr=e85RsqlFHn4sfcj#R(-&@r=ROr<}GkLRL^x-j=gA^9pEXJucgUbiiNd3JA- z@)YfF4Eubs-CspNAV&2l26}2d3*FaC9HaENC?S5X4DRLTzp62Eh@f5nSpHUT2Ujp& zemay2evW4IZ1nQ!%amN|I~iD0UH^#SaEPF!U4|0^dwqi|cqbQ{sngY%|a zTe1Cd*2+p*1c`|HZ38gOC2_Y|*=(L=^ZbKZoraDv3(dR*KInQ~WIxc0U)k2st-8pt zkA4f5wT*-4bfJfJQogTOIsv;BzYR;FhifsN`VyMc1w+w)NrFc!uX{!^XE=_aKdz?; z3Z^x((5V?zXxYKTVVFtGV)vbeQ6Ht}qb^$)s;0WDrt%Cwlo1gGU4B&}0$VL)kDKF& zna99LN@{9!JaK`XG+?a}2b^F985YQUU5ySvZSQ{fk8a@#tlQvrhX01|V_KAWf}$?! zbV=5HV?}Swa1baxG$c`yV3;SH@+no705)!fgcCkk;_bKMQYr)I+@@8Nv-9^OSP}DO z$=^4E!U)imzykYpRF`w5-~fw)NTRg+j{Q~&CZuIDr`1c&*n(!)8Ypk2e9Yss<|*Af z6~rG_GYXmA-dT??@k#xUk zNGKKXV`Pp+;xbjztw;=Q6k{;r!-{6yhUxN;s!}a|9uEW9H`7k5`FR>3M8T4^(PX z-MV8b;r!BvE<`QuDz(Bh)5=2xJH|CJ+WrD!R8uF?{1x{nX!Ts{Pf9qSv(7c8{`rja zUXjreM29q_rQ%esBY$plu|C}j-{|*gKT7KKy;SEWmEXOXmqP_d5J!mA&!_nj@94cI zM@gNwhI132)Lm=UI!d)oitE|=&v8Fz?z-elSf^(#ykKrV9jT{lw2+;$Clr=YAUg9Q zU+gh7TW|g=x}2Ae*qzOn$Y&l4d_OE^qo&kZaOpe?cU7)7k%yb}mY)_dJM7CKm4>94FEJ`TYO;!Z@a5rrP0#UoYg--sqCfvt78IN%Vl~X_FoV zHZ5T1U)DE68()}c3-`Gl7H-`eK_}RRTwfm3L;T-LQyGaNhbIcPO%ei5;7UH$jd|x zUGsP~l}dOPjv4ju&1#C4JE1cFO+QqUp^qhM$Ca$1{+!ewu7D18vT9RoPnueurwjPD+4!5G=Gj*orOo=sy_f+N0tdXvQFdpBh;<$+NS7K z)Fpzumsk%+56fN{;Se- zd#F9vJmO+`0mQ^xGTWFWfr@l3)q1hIZ!=(6+Sa{rlW&(E9Uwq(Ij_PSBF73Hb;IX|%@ZO%KvN)Sj#2X<< zF}YL4Q}Kgvx8ApsN=AIjLbF-(Gq6nbB|SYYz`4H`|N7g1{9ox@SdZs(3++p-Oc3?k z`cB8E6nY&o+)sca=!6hN+oy@5mQBR#3Kx@zs0ezBnk4Z2WcMXk^DHui?0?%#uXB`B zjOVtE?Zytif2Qyv%vvpjTeA>B0*XA1D-%@vRpJy0gm(UyRUfwL%m)C27c{jCn?ZS~ z`vnPaL>8=Id59c&RszZe?Enow!($D#Bmqymip816M$V16r$z;YJohHP)hHA(pAHCfjSWHOdup-dgu6``7D*cw^ zp<;=7`pkPXNX-L>t{M1dsPbR7JNIDg#Q|L0o`;wts-~doV@0^JSCX@9#2v-E;*Vbv zee~;N>1qyjSVt67??(R`>RbNE?&giyqm)_XdJu1Jn6-wC9T7=cox{x(9soRDCm(%w zylXe_WlvjOo9j=hvvAmgOffg7l^omuR$w)h1jE*b0&1jP?l@OolUT2*pdS6CBTuQb zM3*t$gj`zVk~(5<;>d!HV%WA#MyMQ}KpB)miwgNnA})N3j{S%|`%H9g`S|h&W6jHg zt+bDYmY!5fC1_F#E!j&aA$?7hLdy|z{}X@Ib%~yDM^f7fltN4I=2P6f`M#3!*F$sc zhuoZXrTUxe;D0jJ&);h|--*vI&Ct%X82g6nAK`=>*X58O`?RHc+o~g~<43Dsf@3K+ zh2}UR^FB&VKEi@VK|}~}Id*&Fb(@;hjkDgWyv&B$Iz&#ehFl4l)E}6@>OmiQmP@Ww z?Es+od4f7VWX+K+RDf<$ERyJ(7ngCaN5q$oJhz3ZL`S#W_yH^kV$eReHHiWHd# zP4B{@^h&Te`b49LTIOL!l`%>tI&1?*_eTfz3)L;;ntwjXIPIQ)V;Hg6oWT0!B=7LW zxqi!AiRpOKXCg4+;<$oVDD89sVX774z}}l-)x!F4;MRNhDzA>d|NWnK( zAxLp9b-j=-LTEV~np_+=LaD}tp06mwIbMsK{Ofod-OE2Ytz)Y4s_GfNem-jU95CP_sd30x%Ub%2Vnxm-$AXWRnn&*NZod= z4Dm8<-hFA55T&%mKqsTLC4eCI*?l7bi5=ZO-mHsYtF~BfEiMF>SNJI`kP}F`$)RXMXc^v(f9b~m zcwxFy?pRf(P-QI(&xPfpe@vGRXn-S!OEMu5t2fB0oEmDbV!f*+J5i~w%zYzP;R`+Q zHAQ+K_AZbT36zS={ZOGB!DQHEnZY^n??Tx;==YYGxP{A&WxOZwZAjRV(}R$)LnX!k zq&Xl9xe07Mk*{X)m2J4XpXY^XMf#;4BX#6>v}?Y>Chbe5gkUR zm*==qY6W4LR$hD4AGeUTf3o^Gx91UaM;5GAQ{z<7H3I)ZosqL658KcVI6nG>uI7lj zLs^GNj>%Z%;Or7{=b0wyaR$*Rr~ZlK-9PciLK0dYS#JE1oM|au%L>Qzo%zFOSAbn%}YNmto>hA%V(q53_~M*n;X*9J^TXL^xkiR?+uP`9vRWYj$N!v&gOB2NsL}Zh z28czO1m5`>)?o&I5>^spN<_>8K?yfLDqupErZe%}Es&^GktD0itB*-lJLoGTZXGpw zu6CkiB{vVj(yL-9i7atl)i%6Z6Fp+GIAzr3Jy^}Ls+^bhWgq$O|IY3F$sPSC$JsBf z_r1NsAH&!lxlp_Gf+RR2R4x;Q6L<4+MD}fX7^a1ap~_|Y2jP}-8X0(K1~A;y{&Gbs zg4p5nT*I=2pl*j4n|%}>mT#nnr&8R9^WjYCJJkP0y!D;yD(l#@d+*jXC(FC+x zZhf~FlJ@|*AS_t6sODK+%0A^bw#x+*WE^N!>^6jmIFyV#bCRt33usO*xh`BcnSa;V zK+Z|uMZ$J%^n_wo;Z}tVjd7#U!li=14|oSIRoXi z{5ibfa6JZg1w{;#q0cOrCD{rUw+Rn(FnuB6+AAt(CYhEDpJ{jhBx=$cD`y&2y4^hk z*X^^^?A-)qOTy@b-gsj--d(OjaMCDSPQ#ZW%t*tj3vrp8*ih#Pi&YGIFaBKAiQ|h6 z*5`{t9S}ktl0nAl9ZK)`I|HMPk~G8=1u%ms#rz8EY$U z`JQd#5J4%Y{HM9mD$0AyKB1KP^lv50x%QR6wj}s0*v; zmdT@QJbr)m$$l>@zsc?=@<*zv4w?RMO?0&Vv=r$APl-mTnA###MXmp}VEasGjxD9k zlW{j|Ad%NLm5(WOd4UPFq*F%BQCf$O>)P~Rn=~efSY4v$fhJBqI4d+-UeKYC5++-u zgzrIyxM(thjFrtrE7N%5qhCXLLH>$?fxK-OfGrsGHFPNtHcSMTOUk295Y3Z;D!}4Y zrDpC_VpKKmV0!GtcqHS+#4DbH>8-8RxW$?Qte+`=-uc^hs!I6%DUW5dq1;k^9bVfG z7hIZ@L$1gBW`P`m@1u}sp$J-8C`B=vJ2*H9bf0CoDq*E&BdJK}pOP4mPEfQk*kdYw zL5t1GbaBcfgep9g^Of2B%baZAs=&e9hLjscShp(a{0OruSPbH$duk!#^1NOAS0y&O zL78?@!>E$hRk6I{B6+rdUVL7HA2|IT>4AoNVDVx%tNQvrD|hFD*^L}!`fWXMLf%dj zY^Kk1f!aEz?oYREL2EA96R_iqrn zMsWD8T*Jb6mu7P(dTHzOgPwx#!}B#w^$(I_;1$1BYZMpypYVO$vbh^2Y^%HVb$x7d z{FS;YaOC%D4gDecQUua2KqCeNAw&=6_QzwJY0r$yrl-~#ZJrK2dHV?Bh+2bfZbGGUG{C8K ziaN+Cy&L*TaOXth(?5_M`8U_k{^?ZPyw~0aO<4}Q^{Q20YDl=JieQ({8FXjTP9w&u zKM3kJCy#5-RVnCfhT%GLDq#IIcm{2j2DRzvK}RYQ7N^4=R00c$w5!CWXemF61oI7+ zr$Bb@1OoNzvB?`3oqb-#!<8sO=w@E(@)W9woyUmnarI=)A-&4ZQgAf!)ixX$GF-RB0|+f#ZzL+1kZp+T*`r!>@eLR;2CaVKvz`A zZ(YK#!TciZ_{LtiyUKk1vrRK{;i=8>&lCj`|K^1y%>w}YTHW{xMNgu$U%I*#(6Ji8 zzF*Iv7P9jG+~iQz6Vu%GhmeH)rDme@*au?I3Wrd7QMZ#2?Tp^~hMMRVe1egpK5t%9 zr}cb-K^l?POf_(@Y_4oC(q@C$>`@W-I_Py>fhq8G#MZelGRM9n;oLNSvAm-@(Hg0; z7xg}!PO5;Caum))lqt)z{E~|GJ2H(?q+-rSgXBh@Okd!{B4v*+E%2CXCpiAT z$L4g4@hKC8^`%r}o2<+r@l8_?jOV9!?@X}a=JA?2Bz^1$TtoD+O`c2=XI~iEKnY_4 zK_TD8pc%q7v}K+nN(@C^b>BRrR#^G1xC7W_;ygB1-ZHJSb=(H)%Z^0WN!_wAeG*gi zTfGJ)Ksqjyr`EW4DFHIFtZbY;O{C~>mUhsKwyrr9ycwc}gfZZ9{O$%SXkoS zhSVy4G%Ve_jxdtU=`CXtOdF0-HxJbGBTo$SS1KfIKE*=_(l7>=T>312bunv(^M?@@$SxM?x*X{|9 zw7Y5TNE+A&!IgwknEjlRloL6Dj}!zX=XfESt* jYRg;D#EY#`nzf=$=?hA(hylC_{PkZi{?!1d4u<~+V@c5{ literal 0 HcmV?d00001 diff --git a/docker/office/bisheng/index.html b/docker/office/bisheng/index.html new file mode 100644 index 000000000..45c5fecb5 --- /dev/null +++ b/docker/office/bisheng/index.html @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/src/backend/bisheng/api/v1/knowledge.py b/src/backend/bisheng/api/v1/knowledge.py index e4bd18b16..706e2f2f2 100644 --- a/src/backend/bisheng/api/v1/knowledge.py +++ b/src/backend/bisheng/api/v1/knowledge.py @@ -131,8 +131,9 @@ async def process_knowledge(*, session.add(db_file) session.commit() session.refresh(db_file) - files.append(db_file) - file_paths.append(filepath) + if not repeat: + files.append(db_file) + file_paths.append(filepath) logger.info(f'fileName={file_name} col={collection_name}') result.append(db_file.copy()) @@ -310,6 +311,7 @@ def delete_knowledge_file(*, # minio minio_client.MinioClient().delete_minio(str(knowledge_file.id)) + minio_client.MinioClient().delete_minio(str(knowledge_file.object_name)) # elastic esvectore_client = decide_vectorstores(collection_name, 'ElasticKeywordsSearch', embeddings) if esvectore_client: @@ -365,11 +367,12 @@ async def addEmbedding(collection_name, model: str, chunk_size: int, separator: # 存储 mysql db_file = session.get(KnowledgeFile, knowledge_file.id) setattr(db_file, 'status', 2) - setattr(db_file, 'object_name', knowledge_file.file_name) - session.add(db_file) - session.flush() # 原文件 object_name_original = f'original/{db_file.id}' + setattr(db_file, 'object_name', object_name_original) + session.add(db_file) + session.flush() + minio_client.MinioClient().upload_minio(object_name_original, path) texts, metadatas = _read_chunk_text(path, knowledge_file.file_name, chunk_size, diff --git a/src/backend/bisheng/cache/utils.py b/src/backend/bisheng/cache/utils.py index bf5e1fa18..4103d5764 100644 --- a/src/backend/bisheng/cache/utils.py +++ b/src/backend/bisheng/cache/utils.py @@ -13,7 +13,7 @@ import requests from appdirs import user_cache_dir from bisheng.settings import settings -from bisheng.utils.minio_client import mino_client, tmp_bucket +from bisheng.utils.minio_client import MinioClient, tmp_bucket CACHE: Dict[str, Any] = {} @@ -177,10 +177,11 @@ def save_uploaded_file(file, folder_name, file_name): # Save the file with the hash as its name if settings.get_knowledge().get('minio'): + minio_client = MinioClient() # 存储oss file_byte = file.read() - mino_client.upload_tmp(file_name, file_byte) - file_path = mino_client.get_share_link(file_name, tmp_bucket) + minio_client.upload_tmp(file_name, file_byte) + file_path = minio_client.get_share_link(file_name, tmp_bucket) else: file_type = md5_name.split('.')[-1] file_path = folder_path / f'{md5_name}.{file_type}' diff --git a/src/backend/bisheng/database/models/message.py b/src/backend/bisheng/database/models/message.py index 7d611f347..9f291e6f2 100644 --- a/src/backend/bisheng/database/models/message.py +++ b/src/backend/bisheng/database/models/message.py @@ -4,7 +4,7 @@ from bisheng.database.models.base import SQLModelSerializable from pydantic import BaseModel -from sqlalchemy import JSON, Column, DateTime, Text, text +from sqlalchemy import JSON, Column, DateTime, String, Text, text from sqlmodel import Field @@ -22,7 +22,7 @@ class MessageBase(SQLModelSerializable): intermediate_steps: Optional[str] = Field(index=False, sa_column=Column(Text), description='过程日志') - files: Optional[str] = Field(index=False, description='上传的文件等') + files: Optional[str] = Field(sa_column=Column(String(length=4096)), description='上传的文件等') # file_access: Optional[bool] = Field(index=False, default=True, description='召回文件是否可以访问') create_time: Optional[datetime] = Field( sa_column=Column(DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP'))) diff --git a/src/backend/bisheng/interface/chains/custom.py b/src/backend/bisheng/interface/chains/custom.py index 2d4b653da..e594a25a6 100644 --- a/src/backend/bisheng/interface/chains/custom.py +++ b/src/backend/bisheng/interface/chains/custom.py @@ -106,9 +106,11 @@ def initialize(cls, llm: BaseLanguageModel, chain_type: str, prompt: BasePromptTemplate = None, + document_prompt: BasePromptTemplate = None, token_max: str = -1): if chain_type == 'stuff': - return load_qa_chain(llm=llm, chain_type=chain_type, prompt=prompt, token_max=token_max) + return load_qa_chain(llm=llm, chain_type=chain_type, prompt=prompt, + token_max=token_max, document_prompt=document_prompt) else: return load_qa_chain(llm=llm, chain_type=chain_type) diff --git a/src/backend/bisheng/main.py b/src/backend/bisheng/main.py index a51d98d25..92b0d3078 100644 --- a/src/backend/bisheng/main.py +++ b/src/backend/bisheng/main.py @@ -60,7 +60,6 @@ def authjwt_exception_handler(request: Request, exc: AuthJWTException): app.include_router(router_rpc) app.on_event('startup')(create_db_and_tables) app.on_event('startup')(setup_llm_caching) - return app diff --git a/src/backend/bisheng/template/frontend_node/chains.py b/src/backend/bisheng/template/frontend_node/chains.py index 3b56e8217..a0b10115f 100644 --- a/src/backend/bisheng/template/frontend_node/chains.py +++ b/src/backend/bisheng/template/frontend_node/chains.py @@ -301,7 +301,14 @@ class CombineDocsChainNode(FrontendNode): name='prompt', display_name='prompt', advanced=False, - info='只对Stuff类型生效') + info='只对Stuff类型生效'), + TemplateField( + field_type='BasePromptTemplate', + required=False, + show=True, + name='document_prompt', + advanced=False, + ) ], ) diff --git a/src/backend/bisheng/utils/minio_client.py b/src/backend/bisheng/utils/minio_client.py index ab193992c..f5a26ad47 100644 --- a/src/backend/bisheng/utils/minio_client.py +++ b/src/backend/bisheng/utils/minio_client.py @@ -95,6 +95,3 @@ def mkdir(self, bucket: str): if self.minio_client: if not self.minio_client.bucket_exists(bucket): self.minio_client.make_bucket(bucket) - - -mino_client = MinioClient() diff --git a/src/bisheng-langchain/bisheng_langchain/chat_models/proxy_llm.py b/src/bisheng-langchain/bisheng_langchain/chat_models/proxy_llm.py index 9a80897af..a973b8aa1 100644 --- a/src/bisheng-langchain/bisheng_langchain/chat_models/proxy_llm.py +++ b/src/bisheng-langchain/bisheng_langchain/chat_models/proxy_llm.py @@ -195,6 +195,8 @@ def _completion_with_retry(**kwargs: Any) -> Any: 'functions': kwargs.get('functions', []) } response = self.client.post(self.elemai_base_url, json=params) + if response.status_code != 200: + raise return response.json() return _completion_with_retry(**kwargs) diff --git a/src/bisheng-langchain/bisheng_langchain/vectorstores/milvus.py b/src/bisheng-langchain/bisheng_langchain/vectorstores/milvus.py new file mode 100644 index 000000000..e69de29bb