add new feature and bug fix (#55)

1. 增加admin账号配置文件初始化
2. 增加bisheng_rt 服务地址文件初始化
3. 增加admin 管理技能模板，优化技能模板
4. 优化docker 启动脚本，增加健康检查等
5. 非常多bug fix

.github/workflows/ci.yml

@@ -53,6 +53,11 @@ jobs:
           twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi
           
       # 构建 backend 并推送到 Docker hub
+      - name: build lock
+        run: | 
+          cd ./src/backend
+          poetry lock
+          cd ../../
       - name: Build backend and push
         id: docker_build_backend
         uses: docker/build-push-action@v2

.github/workflows/release.yml

@@ -9,6 +9,9 @@ on:
 
 env:
   DOCKERHUB_REPO: dataelement/
+  PY_NEXUS: 110.16.193.170:50083
+  DOCKER_NEXUS: 110.16.193.170:50080
+  RELEASE_VERSION: 0.1.1 # bisheng-langchain nexus 始终替换版本
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -41,18 +44,44 @@ jobs:
           pip install twine
           cd ./src/bisheng-langchain
           python setup.py bdist_wheel
-          repo="http://110.16.193.170:50083/repository/pypi-hosted/"
-          twine upload --verbose -u ${{ secrets.NEXUS_USER }}  -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo  dist/*.whl
+          repo="http://${{ env.PY_NEXUS }}/repository/pypi-hosted/"
+          twine upload --verbose -u ${{ secrets.NEXUS_USER }}  -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo  dist/*.whl    
+      
+      # 发布到 私有仓库
+      - name: set insecure registry
+        run: |
+          echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json
+          sudo service docker restart
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Login Nexus Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: http://${{ env.DOCKER_NEXUS }}/
+          username: ${{ secrets.NEXUS_USER }}
+          password: ${{ secrets.NEXUS_PASSWORD }}
 
-      # 发布到
-      - name: Login
+      - name: Login docker hub
         uses: docker/login-action@v2
         with:
-          # GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
-          # DOCKERHUB_USERNAME 是 docker hub 账号名.
-          # DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # 替换poetry编译为私有服务
+      - name: replace self-host repo
+        uses: snok/install-poetry@v1
+        with:
+          installer-parallel: true
+
+      - name: build lock
+        run: | 
+          cd ./src/backend
+          poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
+          poetry lock
+          cd ../../
+     
       # 构建 backend 并推送到 Docker hub
       - name: Build backend and push
         id: docker_build_backend
@@ -68,6 +97,7 @@ jobs:
             APP_VERSION="release"
           # 生成两个 docker tag: ${APP_VERSION} 和 latest
           tags: |
+            ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:release
             ${{ env.DOCKERHUB_REPO }}bisheng-backend:release
       # 构建 Docker frontend 并推送到 Docker hub
       - name: Build frontend and push
@@ -84,6 +114,10 @@ jobs:
             APP_VERSION="release"
           # 生成两个 docker tag: ${APP_VERSION} 和 latest
           tags: |
+            ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:release
             ${{ env.DOCKERHUB_REPO }}bisheng-frontend:release
 
-     
+      - name: notify feishu
+        uses: sozo-design/curl@v1.0.2
+        with:
+          args: https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b -d '{"msg_type":"text","content":{"text":"release 编译成功"}}'

.vscode/launch.json

@@ -17,7 +17,8 @@
             "jinja": true,
             "justMyCode": false,
             "env": {
-                "PYTHONPATH": "${workspaceRoot}/src/backend/"
+                "PYTHONPATH": "${workspaceRoot}/src/backend/",
+                "config": "/Users/huangly/Code/bisheng/src/backend/bisheng/config.dev.yaml"
             },
             "cwd": "${workspaceFolder}/src/backend/",
         }

docker/bisheng/config/config.yaml

@@ -4,6 +4,16 @@ database_url:
 redis_url:
   "redis:6379"
 
+# admin 用户配置
+admin:
+  user_name: "admin"
+  password: "1234"
+
+# bisheng-rt服务地址
+# bisheng_rt: 
+#   name: "RT-Server"
+#   server: "192.168.0.1:9001"
+
 # 为知识库的embedding进行模型撇脂
 knowledges:
   embeddings:
@@ -15,7 +25,10 @@ knowledges:
     # Milvus 最低要求cpu 4C 8G 推荐4C 16G
     Milvus: # 如果需要切换其他vectordb，确保其他服务已经启动，然后配置对应参数
       connection_args: {'host': '110.16.193.170', 'port': '50032', 'user': '', 'password': '', 'secure': False}
-
+    ElasticKeywordsSearch:
+      elasticsearch_url: 'https://192.168.106.14:9200'
+      ssl_verify: {'ca_certs': False, 'basic_auth': "('elastic', 'F94h5JtdQn6EQB-G9Hjv')", 'verify_certs': False}
+
 agents:
   ZeroShotAgent:
     documentation: "https://python.langchain.com/docs/modules/agents/how_to/custom_mrkl_agent"
@@ -32,6 +45,10 @@ agents:
   SQLAgent:
     documentation: ""
 chains:
+  MultiRetrievalQA:
+    documentation: ""
+  SequentialChain:
+    documentation: ""
   LLMChain:
     documentation: "https://python.langchain.com/docs/modules/chains/foundational/llm_chain"
   LLMMathChain:

docker/docker-compose.yml

@@ -25,6 +25,12 @@ services:
 
   backend:
     image: dataelement/bisheng-backend:latest
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7860/health"]
+      interval: 1m30s
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     volumes:
       - ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/config/config.yaml:/app/bisheng/config.yaml
       - ${DOCKER_VOLUME_DIRECTORY:-.}/bisheng/data/:/app/data/
@@ -34,6 +40,9 @@ services:
       - seccomp:unconfined
     command: bash -c "uvicorn bisheng.main:app --host 0.0.0.0 --port 7860 --workers 2" # --workers 表示使用几个进程，提高并发度
     restart: on-failure
+    depends_on:
+      - "mysql"
+      - "redis"
 
   nginx:
     image: dataelement/bisheng-frontend:latest

docker/mysql/conf/my.cnf

@@ -1,13 +1,13 @@
 [client]
-default-character-set=utf8
+default-character-set=utf8mb4
 
 [mysql]
-default-character-set=utf8
+default-character-set=utf8mb4
 
 [mysqld]
-init_connect='SET collation_connection = utf8_unicode_ci'
-init_connect='SET NAMES utf8'
-character-set-server=utf8
-collation-server=utf8_unicode_ci
+init_connect='SET collation_connection = utf8mb4_unicode_ci'
+init_connect='SET NAMES utf8mb4'
+character-set-server=utf8mb4
+collation-server=utf8mb4_unicode_ci
 skip-character-set-client-handshake
 sql_mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION

src/backend/Dockerfile

@@ -20,4 +20,4 @@ RUN python -m pip install --upgrade pip && \
 RUN poetry config virtualenvs.create false
 RUN poetry install --no-interaction --no-ansi --without dev
 
-CMD ["uvicorn", "bisheng.main:app", "--workers", "20",  "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "bisheng.main:app", "--workers", "2",  "--host", "0.0.0.0", "--port", "7860"]

src/backend/bisheng/api/utils.py

@@ -6,9 +6,7 @@
 
 
 def has_api_terms(word: str):
-    return 'api' in word and (
-        'key' in word or ('token' in word and 'tokens' not in word)
-    )
+    return 'api' in word and ('key' in word or ('token' in word and 'tokens' not in word))
 
 
 def remove_api_keys(flow: dict):
@@ -18,11 +16,8 @@ def remove_api_keys(flow: dict):
             node_data = node.get('data').get('node')
             template = node_data.get('template')
             for value in template.values():
-                if (
-                    isinstance(value, dict)
-                    and has_api_terms(value['name'])
-                    and value.get('password')
-                ):
+                if (isinstance(value, dict) and has_api_terms(value['name']) and
+                        value.get('password')):
                     value['value'] = None
 
     return flow
@@ -32,7 +27,9 @@ def build_input_keys_response(langchain_object, artifacts):
     """Build the input keys response."""
 
     input_keys_response = {
-        'input_keys': {key: '' for key in langchain_object.input_keys},
+        'input_keys': {
+            key: '' for key in langchain_object.input_keys
+        },
         'memory_keys': [],
         'handle_keys': artifacts.get('handle_keys', []),
     }
@@ -43,9 +40,7 @@ def build_input_keys_response(langchain_object, artifacts):
             input_keys_response['input_keys'][key] = value
     # If the object has memory, that memory will have a memory_variables attribute
     # memory variables should be removed from the input keys
-    if hasattr(langchain_object, 'memory') and hasattr(
-        langchain_object.memory, 'memory_variables'
-    ):
+    if hasattr(langchain_object, 'memory') and hasattr(langchain_object.memory, 'memory_variables'):
         # Remove memory variables from input keys
         input_keys_response['input_keys'] = {
             key: value
@@ -55,9 +50,7 @@ def build_input_keys_response(langchain_object, artifacts):
         # Add memory variables to memory_keys
         input_keys_response['memory_keys'] = langchain_object.memory.memory_variables
 
-    if hasattr(langchain_object, 'prompt') and hasattr(
-        langchain_object.prompt, 'template'
-    ):
+    if hasattr(langchain_object, 'prompt') and hasattr(langchain_object.prompt, 'template'):
         input_keys_response['template'] = langchain_object.prompt.template
 
     return input_keys_response
@@ -81,17 +74,17 @@ def build_flow(graph_data: dict, artifacts, process_file=False, flow_id=None, ch
                 'log': f'Building node {vertex.vertex_type}',
             }
             yield str(StreamData(event='log', data=log_dict))
-            # 如果存在文件，当前不操作文件，避免重复操作
-            if not process_file:
-                template_dict = {
-                    key: value
-                    for key, value in vertex.data['node']['template'].items()
-                    if isinstance(value, dict)
-                }
-                for key, value in template_dict.items():
-                    if value.get('type') == 'file':
-                        # 过滤掉文件
-                        vertex.params[key] = ''
+            # # 如果存在文件，当前不操作文件，避免重复操作
+            # if not process_file and chat_id is not None:
+            #     template_dict = {
+            #         key: value
+            #         for key, value in vertex.data['node']['template'].items()
+            #         if isinstance(value, dict)
+            #     }
+            #     for key, value in template_dict.items():
+            #         if value.get('type') == 'file':
+            #             # 过滤掉文件
+            #             vertex.params[key] = ''
 
             # vectore store 引入自动建库逻辑
             # 聊天窗口等flow 主动生成的vector 需要新建临时collection
@@ -104,8 +97,7 @@ def build_flow(graph_data: dict, artifacts, process_file=False, flow_id=None, ch
             params = vertex._built_object_repr()
             valid = True
             logger.debug(
-                f"Building node {str(params)[:50]}{'...' if len(str(params)) > 50 else ''}"
-            )
+                f"Building node {str(params)[:50]}{'...' if len(str(params)) > 50 else ''}")
             if vertex.artifacts:
                 # The artifacts will be prompt variables
                 # passed to build_input_keys_response
@@ -133,40 +125,44 @@ def build_flow(graph_data: dict, artifacts, process_file=False, flow_id=None, ch
     return graph
 
 
-def build_flow_no_yield(graph_data: dict, artifacts, process_file=False, flow_id=None, chat_id=None):
+def build_flow_no_yield(graph_data: dict,
+                        artifacts,
+                        process_file=False,
+                        flow_id=None,
+                        chat_id=None):
     try:
         # Some error could happen when building the graph
         graph = Graph.from_payload(graph_data)
     except Exception as exc:
         logger.exception(exc)
-        return
+        raise exc
 
     for i, vertex in enumerate(graph.generator_build(), 1):
         try:
-            # 如果存在文件，当前不操作文件，避免重复操作
-            if not process_file:
-                template_dict = {
-                    key: value
-                    for key, value in vertex.data['node']['template'].items()
-                    if isinstance(value, dict)
-                }
-                for key, value in template_dict.items():
-                    if value.get('type') == 'file':
-                        # 过滤掉文件
-                        vertex.params[key] = ''
+            # # 如果存在文件，当前不操作文件，避免重复操作
+            # if not process_file:
+            #     template_dict = {
+            #         key: value
+            #         for key, value in vertex.data['node']['template'].items()
+            #         if isinstance(value, dict)
+            #     }
+            #     for key, value in template_dict.items():
+            #         if value.get('type') == 'file':
+            #             # 过滤掉文件
+            #             vertex.params[key] = ''
 
             # vectore store 引入自动建库逻辑
             # 聊天窗口等flow 主动生成的vector 需要新建临时collection
             # tmp_{chat_id}
             if vertex.base_type == 'vectorstores':
                 if 'collection_name' in vertex.params and not vertex.params.get('collection_name'):
                     vertex.params['collection_name'] = f'tmp_{flow_id}_{chat_id}'
+                    logger.info(f"rename_vector_col col={vertex.params['collection_name']}")
 
             vertex.build()
             params = vertex._built_object_repr()
             logger.debug(
-                f"Building node {str(params)[:50]}{'...' if len(str(params)) > 50 else ''}"
-            )
+                f"Building node {str(params)[:50]}{'...' if len(str(params)) > 50 else ''}")
             if vertex.artifacts:
                 # The artifacts will be prompt variables
                 # passed to build_input_keys_response