Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/ZuzooVn/dify
Browse files Browse the repository at this point in the history
  • Loading branch information
ZuzooVn committed Jul 12, 2024
2 parents 83a689f + 066168d commit c193361
Show file tree
Hide file tree
Showing 63 changed files with 1,267 additions and 662 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/api-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ jobs:
- name: Run Workflow
run: poetry run -C api bash dev/pytest/pytest_workflow.sh

- name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
- name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale)
uses: hoverkraft-tech/compose-action@v2.0.0
with:
compose-file: |
Expand All @@ -89,5 +89,6 @@ jobs:
pgvecto-rs
pgvector
chroma
myscale
- name: Test Vector Stores
run: poetry run -C api bash dev/pytest/pytest_vdb.sh
10 changes: 9 additions & 1 deletion api/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ OCI_REGION=your-region
WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*

# Vector database configuration, support: weaviate, qdrant, milvus, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector
# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector
VECTOR_STORE=weaviate

# Weaviate configuration
Expand All @@ -106,6 +106,14 @@ MILVUS_USER=root
MILVUS_PASSWORD=Milvus
MILVUS_SECURE=false

# MyScale configuration
MYSCALE_HOST=127.0.0.1
MYSCALE_PORT=8123
MYSCALE_USER=default
MYSCALE_PASSWORD=
MYSCALE_DATABASE=default
MYSCALE_FTS_PARAMS=

# Relyt configuration
RELYT_HOST=127.0.0.1
RELYT_PORT=5432
Expand Down
2 changes: 2 additions & 0 deletions api/configs/middleware/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
from configs.middleware.vdb.chroma_config import ChromaConfig
from configs.middleware.vdb.milvus_config import MilvusConfig
from configs.middleware.vdb.myscale_config import MyScaleConfig
from configs.middleware.vdb.opensearch_config import OpenSearchConfig
from configs.middleware.vdb.oracle_config import OracleConfig
from configs.middleware.vdb.pgvector_config import PGVectorConfig
Expand Down Expand Up @@ -187,6 +188,7 @@ class MiddlewareConfig(
AnalyticdbConfig,
ChromaConfig,
MilvusConfig,
MyScaleConfig,
OpenSearchConfig,
OracleConfig,
PGVectorConfig,
Expand Down
39 changes: 39 additions & 0 deletions api/configs/middleware/vdb/myscale_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
from typing import Optional

from pydantic import BaseModel, Field, PositiveInt


class MyScaleConfig(BaseModel):
"""
MyScale configs
"""

MYSCALE_HOST: Optional[str] = Field(
description='MyScale host',
default=None,
)

MYSCALE_PORT: Optional[PositiveInt] = Field(
description='MyScale port',
default=8123,
)

MYSCALE_USER: Optional[str] = Field(
description='MyScale user',
default=None,
)

MYSCALE_PASSWORD: Optional[str] = Field(
description='MyScale password',
default=None,
)

MYSCALE_DATABASE: Optional[str] = Field(
description='MyScale database name',
default=None,
)

MYSCALE_FTS_PARAMS: Optional[str] = Field(
description='MyScale fts index parameters',
default=None,
)
4 changes: 2 additions & 2 deletions api/controllers/console/datasets/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@ def get(self):
RetrievalMethod.SEMANTIC_SEARCH
]
}
case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB:
case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE:
return {
'retrieval_method': [
RetrievalMethod.SEMANTIC_SEARCH,
Expand All @@ -572,7 +572,7 @@ def get(self, vector_type):
RetrievalMethod.SEMANTIC_SEARCH
]
}
case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB:
case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE:
return {
'retrieval_method': [
RetrievalMethod.SEMANTIC_SEARCH,
Expand Down
9 changes: 5 additions & 4 deletions api/controllers/inner_api/wraps.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,22 @@
from hashlib import sha1
from hmac import new as hmac_new

from flask import abort, current_app, request
from flask import abort, request

from configs import dify_config
from extensions.ext_database import db
from models.model import EndUser


def inner_api_only(view):
@wraps(view)
def decorated(*args, **kwargs):
if not current_app.config['INNER_API']:
if not dify_config.INNER_API:
abort(404)

# get header 'X-Inner-Api-Key'
inner_api_key = request.headers.get('X-Inner-Api-Key')
if not inner_api_key or inner_api_key != current_app.config['INNER_API_KEY']:
if not inner_api_key or inner_api_key != dify_config.INNER_API_KEY:
abort(404)

return view(*args, **kwargs)
Expand All @@ -28,7 +29,7 @@ def decorated(*args, **kwargs):
def inner_api_user_auth(view):
@wraps(view)
def decorated(*args, **kwargs):
if not current_app.config['INNER_API']:
if not dify_config.INNER_API:
return view(*args, **kwargs)

# get header 'X-Inner-Api-Key'
Expand Down
4 changes: 2 additions & 2 deletions api/controllers/service_api/app/app.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

from flask import current_app
from flask_restful import Resource, fields, marshal_with

from configs import dify_config
from controllers.service_api import api
from controllers.service_api.app.error import AppUnavailableError
from controllers.service_api.wraps import validate_app_token
Expand Down Expand Up @@ -78,7 +78,7 @@ def get(self, app_model: App):
"transfer_methods": ["remote_url", "local_file"]
}}),
'system_parameters': {
'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT')
'image_file_size_limit': dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT
}
}

Expand Down
4 changes: 2 additions & 2 deletions api/controllers/service_api/index.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from flask import current_app
from flask_restful import Resource

from configs import dify_config
from controllers.service_api import api


Expand All @@ -9,7 +9,7 @@ def get(self):
return {
"welcome": "Dify OpenAPI",
"api_version": "v1",
"server_version": current_app.config['CURRENT_VERSION']
"server_version": dify_config.CURRENT_VERSION,
}


Expand Down
4 changes: 2 additions & 2 deletions api/controllers/web/app.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from flask import current_app
from flask_restful import fields, marshal_with

from configs import dify_config
from controllers.web import api
from controllers.web.error import AppUnavailableError
from controllers.web.wraps import WebApiResource
Expand Down Expand Up @@ -75,7 +75,7 @@ def get(self, app_model: App, end_user):
"transfer_methods": ["remote_url", "local_file"]
}}),
'system_parameters': {
'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT')
'image_file_size_limit': dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT
}
}

Expand Down
4 changes: 2 additions & 2 deletions api/controllers/web/site.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@

from flask import current_app
from flask_restful import fields, marshal_with
from werkzeug.exceptions import Forbidden

from configs import dify_config
from controllers.web import api
from controllers.web.wraps import WebApiResource
from extensions.ext_database import db
Expand Down Expand Up @@ -84,7 +84,7 @@ def __init__(self, tenant, app, site, end_user, can_replace_logo):
self.can_replace_logo = can_replace_logo

if can_replace_logo:
base_url = current_app.config.get('FILES_URL')
base_url = dify_config.FILES_URL
remove_webapp_brand = tenant.custom_config_dict.get('remove_webapp_brand', False)
replace_webapp_logo = f'{base_url}/files/workspaces/{tenant.id}/webapp-logo' if tenant.custom_config_dict.get('replace_webapp_logo') else None
self.custom_config = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ features:
- agent-thought
model_properties:
mode: chat
context_size: 32000
context_size: 128000
parameter_rules:
- name: temperature
use_template: temperature
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ features:
- agent-thought
model_properties:
mode: chat
context_size: 32000
context_size: 128000
parameter_rules:
- name: temperature
use_template: temperature
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -616,30 +616,34 @@ def _convert_prompt_message_to_dict(self, message: PromptMessage, credentials: O
message = cast(AssistantPromptMessage, message)
message_dict = {"role": "assistant", "content": message.content}
if message.tool_calls:
# message_dict["tool_calls"] = [helper.dump_model(PromptMessageFunction(function=tool_call)) for tool_call
# in
# message.tool_calls]

function_call = message.tool_calls[0]
message_dict["function_call"] = {
"name": function_call.function.name,
"arguments": function_call.function.arguments,
}
function_calling_type = credentials.get('function_calling_type', 'no_call')
if function_calling_type == 'tool_call':
message_dict["tool_calls"] = [tool_call.dict() for tool_call in
message.tool_calls]
elif function_calling_type == 'function_call':
function_call = message.tool_calls[0]
message_dict["function_call"] = {
"name": function_call.function.name,
"arguments": function_call.function.arguments,
}
elif isinstance(message, SystemPromptMessage):
message = cast(SystemPromptMessage, message)
message_dict = {"role": "system", "content": message.content}
elif isinstance(message, ToolPromptMessage):
message = cast(ToolPromptMessage, message)
# message_dict = {
# "role": "tool",
# "content": message.content,
# "tool_call_id": message.tool_call_id
# }
message_dict = {
"role": "tool" if credentials and credentials.get('function_calling_type', 'no_call') == 'tool_call' else "function",
"content": message.content,
"name": message.tool_call_id
}
function_calling_type = credentials.get('function_calling_type', 'no_call')
if function_calling_type == 'tool_call':
message_dict = {
"role": "tool",
"content": message.content,
"tool_call_id": message.tool_call_id
}
elif function_calling_type == 'function_call':
message_dict = {
"role": "function",
"content": message.content,
"name": message.tool_call_id
}
else:
raise ValueError(f"Got unknown type {message}")

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
model: ernie-4.0-turbo-8k-preview
label:
en_US: Ernie-4.0-turbo-8k-preview
model_type: llm
features:
- agent-thought
model_properties:
mode: chat
context_size: 8192
parameter_rules:
- name: temperature
use_template: temperature
min: 0.1
max: 1.0
default: 0.8
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 2
max: 2048
- name: presence_penalty
use_template: presence_penalty
default: 1.0
min: 1.0
max: 2.0
- name: frequency_penalty
use_template: frequency_penalty
- name: response_format
use_template: response_format
- name: disable_search
label:
zh_Hans: 禁用搜索
en_US: Disable Search
type: boolean
help:
zh_Hans: 禁用模型自行进行外部搜索。
en_US: Disable the model to perform external search.
required: false
7 changes: 3 additions & 4 deletions api/core/rag/datasource/keyword/keyword_factory.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from typing import Any

from flask import current_app

from configs import dify_config
from core.rag.datasource.keyword.jieba.jieba import Jieba
from core.rag.datasource.keyword.keyword_base import BaseKeyword
from core.rag.models.document import Document
Expand All @@ -14,8 +13,8 @@ def __init__(self, dataset: Dataset):
self._keyword_processor = self._init_keyword()

def _init_keyword(self) -> BaseKeyword:
config = current_app.config
keyword_type = config.get('KEYWORD_STORE')
config = dify_config
keyword_type = config.KEYWORD_STORE

if not keyword_type:
raise ValueError("Keyword store must be specified.")
Expand Down
Empty file.
Loading

0 comments on commit c193361

Please sign in to comment.