Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core): AWEL flow 2.0 backend code #1874

Closed
wants to merge 19 commits into from
Closed

feat(core): AWEL flow 2.0 backend code #1874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
8372b13
feat(core): Add UI component for AWEL flow
fangyinc Jul 30, 2024
d6477da
feat: Add flow2.0 examples
fangyinc Aug 5, 2024
97b57fb
feat(core): Support refresh for AWEL flow
fangyinc Aug 6, 2024
22c3d73
feat(core): Support dag scope variables
fangyinc Aug 22, 2024
66756ea
feat(core): Support complex variables parsing
fangyinc Aug 11, 2024
d4f1afc
feat(core): Support dag scope variables
fangyinc Aug 22, 2024
25f6d8a
chore: Merge latest code
fangyinc Aug 22, 2024
4f2c56d
feat(core): Add debug and export/import for AWEL flow
fangyinc Aug 17, 2024
6442920
chore: Merge latest code
fangyinc Aug 22, 2024
44a1445
chore: Merge latest code
fangyinc Aug 22, 2024
354a8ac
fix(core): Fix upload ui component attr error
fangyinc Aug 18, 2024
85369a5
feat(core): Add multi-instance config
fangyinc Aug 19, 2024
1c7a6c9
feat(core): Add code editor for UI component
fangyinc Aug 19, 2024
de702ef
feat(core): Support dag scope variables
fangyinc Aug 22, 2024
3d8e6e4
chore: Modify database key
fangyinc Aug 22, 2024
6507aef
feat(core): Support AWEL more upload examples
fangyinc Aug 22, 2024
9b14e78
feat(app): fix sqlite app add bug
yhjun1026 Aug 22, 2024
eab1b5f
chore: Add upgrade DDL
fangyinc Aug 23, 2024
0c03041
refactor: upgrade gpt-vis (#1871)
hustcc Aug 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .env.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,17 @@ DBGPT_LOG_LEVEL=INFO
# API_KEYS - The list of API keys that are allowed to access the API. Each of the below are an option, separated by commas.
# API_KEYS=dbgpt

#*******************************************************************#
#** ENCRYPT **#
#*******************************************************************#
# ENCRYPT KEY - The key used to encrypt and decrypt the data
# ENCRYPT_KEY=your_secret_key

#*******************************************************************#
#** File Server **#
#*******************************************************************#
## The local storage path of the file server, the default is pilot/data/file_server
# FILE_SERVER_LOCAL_STORAGE_PATH =

#*******************************************************************#
#** Application Config **#
Expand Down
3 changes: 3 additions & 0 deletions .mypy.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,6 @@ ignore_missing_imports = True

[mypy-networkx.*]
ignore_missing_imports = True

[mypy-pypdf.*]
ignore_missing_imports = True
129 changes: 116 additions & 13 deletions assets/schema/dbgpt.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@ CREATE TABLE IF NOT EXISTS `knowledge_document`
`id` int NOT NULL AUTO_INCREMENT COMMENT 'auto increment id',
`doc_name` varchar(100) NOT NULL COMMENT 'document path name',
`doc_type` varchar(50) NOT NULL COMMENT 'doc type',
`doc_token` varchar(100) NOT NULL COMMENT 'doc token',
`space` varchar(50) NOT NULL COMMENT 'knowledge space',
`chunk_size` int NOT NULL COMMENT 'chunk size',
`last_sync` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT 'last sync time',
`status` varchar(50) NOT NULL COMMENT 'status TODO,RUNNING,FAILED,FINISHED',
`content` LONGTEXT NOT NULL COMMENT 'knowledge embedding sync result',
`result` TEXT NULL COMMENT 'knowledge content',
`questions` TEXT NULL COMMENT 'document related questions',
`vector_ids` LONGTEXT NULL COMMENT 'vector_ids',
`summary` LONGTEXT NULL COMMENT 'knowledge summary',
`gmt_created` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
Expand All @@ -53,6 +55,7 @@ CREATE TABLE IF NOT EXISTS `document_chunk`
`doc_type` varchar(50) NOT NULL COMMENT 'doc type',
`document_id` int NOT NULL COMMENT 'document parent id',
`content` longtext NOT NULL COMMENT 'chunk content',
`questions` text NULL COMMENT 'chunk related questions',
`meta_info` varchar(200) NOT NULL COMMENT 'metadata info',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
Expand All @@ -61,7 +64,6 @@ CREATE TABLE IF NOT EXISTS `document_chunk`
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='knowledge document chunk detail';



CREATE TABLE IF NOT EXISTS `connect_config`
(
`id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
Expand All @@ -74,6 +76,9 @@ CREATE TABLE IF NOT EXISTS `connect_config`
`db_pwd` varchar(255) DEFAULT NULL COMMENT 'db password',
`comment` text COMMENT 'db comment',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`user_name` varchar(255) DEFAULT NULL COMMENT 'user name',
`user_id` varchar(255) DEFAULT NULL COMMENT 'user id',

PRIMARY KEY (`id`),
UNIQUE KEY `uk_db` (`db_name`),
KEY `idx_q_db_type` (`db_type`)
Expand All @@ -88,11 +93,13 @@ CREATE TABLE IF NOT EXISTS `chat_history`
`user_name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'interlocutor',
`messages` text COLLATE utf8mb4_unicode_ci COMMENT 'Conversation details',
`message_ids` text COLLATE utf8mb4_unicode_ci COMMENT 'Message id list, split by comma',
`app_code` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'App unique code',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
UNIQUE KEY `conv_uid` (`conv_uid`),
PRIMARY KEY (`id`)
PRIMARY KEY (`id`),
KEY `idx_chat_his_app_code` (`app_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT 'Chat history';

CREATE TABLE IF NOT EXISTS `chat_history_message`
Expand All @@ -108,6 +115,7 @@ CREATE TABLE IF NOT EXISTS `chat_history_message`
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT 'Chat history message';


CREATE TABLE IF NOT EXISTS `chat_feed_back`
(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
Expand All @@ -118,6 +126,11 @@ CREATE TABLE IF NOT EXISTS `chat_feed_back`
`question` longtext DEFAULT NULL COMMENT 'User question',
`knowledge_space` varchar(128) DEFAULT NULL COMMENT 'Knowledge space name',
`messages` longtext DEFAULT NULL COMMENT 'The details of user feedback',
`message_id` varchar(255) NULL COMMENT 'Message id',
`feedback_type` varchar(50) NULL COMMENT 'Feedback type like or unlike',
`reason_types` varchar(255) NULL COMMENT 'Feedback reason categories',
`remark` text NULL COMMENT 'Feedback remark',
`user_code` varchar(128) NULL COMMENT 'User code',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
Expand Down Expand Up @@ -167,17 +180,20 @@ CREATE TABLE IF NOT EXISTS `plugin_hub`
CREATE TABLE IF NOT EXISTS `prompt_manage`
(
`id` int(11) NOT NULL AUTO_INCREMENT,
`chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Chat scene',
`sub_chat_scene` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Sub chat scene',
`prompt_type` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt type: common or private',
`prompt_name` varchar(256) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'prompt name',
`content` longtext COLLATE utf8mb4_unicode_ci COMMENT 'Prompt content',
`input_variables` varchar(1024) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt input variables(split by comma))',
`model` varchar(128) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt model name(we can use different models for different prompt)',
`prompt_language` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt language(eg:en, zh-cn)',
`prompt_format` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT 'f-string' COMMENT 'Prompt format(eg: f-string, jinja2)',
`prompt_desc` varchar(512) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'Prompt description',
`user_name` varchar(128) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'User name',
`chat_scene` varchar(100) DEFAULT NULL COMMENT 'Chat scene',
`sub_chat_scene` varchar(100) DEFAULT NULL COMMENT 'Sub chat scene',
`prompt_type` varchar(100) DEFAULT NULL COMMENT 'Prompt type: common or private',
`prompt_name` varchar(256) DEFAULT NULL COMMENT 'prompt name',
`prompt_code` varchar(256) DEFAULT NULL COMMENT 'prompt code',
`content` longtext COMMENT 'Prompt content',
`input_variables` varchar(1024) DEFAULT NULL COMMENT 'Prompt input variables(split by comma))',
`response_schema` text DEFAULT NULL COMMENT 'Prompt response schema',
`model` varchar(128) DEFAULT NULL COMMENT 'Prompt model name(we can use different models for different prompt)',
`prompt_language` varchar(32) DEFAULT NULL COMMENT 'Prompt language(eg:en, zh-cn)',
`prompt_format` varchar(32) DEFAULT 'f-string' COMMENT 'Prompt format(eg: f-string, jinja2)',
`prompt_desc` varchar(512) DEFAULT NULL COMMENT 'Prompt description',
`user_code` varchar(128) DEFAULT NULL COMMENT 'User code',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'created time',
`gmt_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'update time',
Expand All @@ -186,6 +202,8 @@ CREATE TABLE IF NOT EXISTS `prompt_manage`
KEY `gmt_created_idx` (`gmt_created`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Prompt management table';



CREATE TABLE IF NOT EXISTS `gpts_conversations` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
`conv_id` varchar(255) NOT NULL COMMENT 'The unique id of the conversation record',
Expand Down Expand Up @@ -232,11 +250,15 @@ CREATE TABLE `gpts_messages` (
`receiver` varchar(255) NOT NULL COMMENT 'Who receive message in the current conversation turn',
`model_name` varchar(255) DEFAULT NULL COMMENT 'message generate model',
`rounds` int(11) NOT NULL COMMENT 'dialogue turns',
`is_success` int(4) NULL DEFAULT 0 COMMENT 'agent message is success',
`app_code` varchar(255) NOT NULL COMMENT 'Current AI assistant code',
`app_name` varchar(255) NOT NULL COMMENT 'Current AI assistant name',
`content` text COMMENT 'Content of the speech',
`current_goal` text COMMENT 'The target corresponding to the current message',
`context` text COMMENT 'Current conversation context',
`review_info` text COMMENT 'Current conversation review info',
`action_report` text COMMENT 'Current conversation action report',
`resource_info` text DEFAULT NULL COMMENT 'Current conversation resource info',
`role` varchar(255) DEFAULT NULL COMMENT 'The role of the current message content',
`created_at` datetime DEFAULT NULL COMMENT 'create time',
`updated_at` datetime DEFAULT NULL COMMENT 'last update time',
Expand Down Expand Up @@ -286,6 +308,7 @@ CREATE TABLE `dbgpt_serve_flow` (
`define_type` varchar(32) null comment 'Flow define type(json or python)',
`label` varchar(128) DEFAULT NULL COMMENT 'Flow label',
`editable` int DEFAULT NULL COMMENT 'Editable, 0: editable, 1: not editable',
`variables` text DEFAULT NULL COMMENT 'Flow variables, JSON format',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_uid` (`uid`),
KEY `ix_dbgpt_serve_flow_sys_code` (`sys_code`),
Expand All @@ -295,6 +318,51 @@ CREATE TABLE `dbgpt_serve_flow` (
KEY `ix_dbgpt_serve_flow_name` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

-- dbgpt.dbgpt_serve_file definition
CREATE TABLE `dbgpt_serve_file` (
`id` int NOT NULL AUTO_INCREMENT COMMENT 'Auto increment id',
`bucket` varchar(255) NOT NULL COMMENT 'Bucket name',
`file_id` varchar(255) NOT NULL COMMENT 'File id',
`file_name` varchar(256) NOT NULL COMMENT 'File name',
`file_size` int DEFAULT NULL COMMENT 'File size',
`storage_type` varchar(32) NOT NULL COMMENT 'Storage type',
`storage_path` varchar(512) NOT NULL COMMENT 'Storage path',
`uri` varchar(512) NOT NULL COMMENT 'File URI',
`custom_metadata` text DEFAULT NULL COMMENT 'Custom metadata, JSON format',
`file_hash` varchar(128) DEFAULT NULL COMMENT 'File hash',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'Record creation time',
`gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Record update time',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_bucket_file_id` (`bucket`, `file_id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

-- dbgpt.dbgpt_serve_variables definition
CREATE TABLE `dbgpt_serve_variables` (
`id` int NOT NULL AUTO_INCREMENT COMMENT 'Auto increment id',
`key` varchar(128) NOT NULL COMMENT 'Variable key',
`name` varchar(128) DEFAULT NULL COMMENT 'Variable name',
`label` varchar(128) DEFAULT NULL COMMENT 'Variable label',
`value` text DEFAULT NULL COMMENT 'Variable value, JSON format',
`value_type` varchar(32) DEFAULT NULL COMMENT 'Variable value type(string, int, float, bool)',
`category` varchar(32) DEFAULT 'common' COMMENT 'Variable category(common or secret)',
`encryption_method` varchar(32) DEFAULT NULL COMMENT 'Variable encryption method(fernet, simple, rsa, aes)',
`salt` varchar(128) DEFAULT NULL COMMENT 'Variable salt',
`scope` varchar(32) DEFAULT 'global' COMMENT 'Variable scope(global,flow,app,agent,datasource,flow_priv,agent_priv, ""etc)',
`scope_key` varchar(256) DEFAULT NULL COMMENT 'Variable scope key, default is empty, for scope is "flow_priv", the scope_key is dag id of flow',
`enabled` int DEFAULT 1 COMMENT 'Variable enabled, 0: disabled, 1: enabled',
`description` text DEFAULT NULL COMMENT 'Variable description',
`user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
`sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
`gmt_created` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'Record creation time',
`gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Record update time',
PRIMARY KEY (`id`),
KEY `ix_your_table_name_key` (`key`),
KEY `ix_your_table_name_name` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;


-- dbgpt.gpts_app definition
CREATE TABLE `gpts_app` (
`id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
Expand All @@ -309,6 +377,9 @@ CREATE TABLE `gpts_app` (
`created_at` datetime DEFAULT NULL COMMENT 'create time',
`updated_at` datetime DEFAULT NULL COMMENT 'last update time',
`icon` varchar(1024) DEFAULT NULL COMMENT 'app icon, url',
`published` varchar(64) DEFAULT 'false' COMMENT 'Has it been published?',
`param_need` text DEFAULT NULL COMMENT 'Parameter information supported by the application',
`admins` text DEFAULT NULL COMMENT 'administrator',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_gpts_app` (`app_name`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
Expand Down Expand Up @@ -363,6 +434,38 @@ CREATE TABLE IF NOT EXISTS `dbgpt_cluster_registry_instance` (
UNIQUE KEY `uk_model_instance` (`model_name`, `host`, `port`, `sys_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='Cluster model instance table, for registering and managing model instances';

-- dbgpt.recommend_question definition
CREATE TABLE `recommend_question` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
`gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
`gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',
`app_code` varchar(255) DEFAULT NULL COMMENT 'Current AI assistant code',
`question` text DEFAULT NULL COMMENT 'question',
`user_code` int(11) NOT NULL COMMENT 'user code',
`sys_code` varchar(255) NOT NULL COMMENT 'system app code',
`valid` varchar(10) DEFAULT 'true' COMMENT 'is it effective，true/false',
`chat_mode` varchar(255) DEFAULT NULL COMMENT 'Conversation scene mode，chat_knowledge...',
`params` text DEFAULT NULL COMMENT 'question param',
`is_hot_question` varchar(10) DEFAULT 'false' COMMENT 'Is it a popular recommendation question?',
PRIMARY KEY (`id`),
KEY `idx_rec_q_app_code` (`app_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT="AI application related recommendation issues";

-- dbgpt.user_recent_apps definition
CREATE TABLE `user_recent_apps` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
`gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
`gmt_modified` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'last update time',
`app_code` varchar(255) DEFAULT NULL COMMENT 'AI assistant code',
`last_accessed` timestamp NULL DEFAULT NULL COMMENT 'User recent usage time',
`user_code` varchar(255) DEFAULT NULL COMMENT 'user code',
`sys_code` varchar(255) DEFAULT NULL COMMENT 'system app code',
PRIMARY KEY (`id`),
KEY `idx_user_r_app_code` (`app_code`),
KEY `idx_last_accessed` (`last_accessed`),
KEY `idx_user_code` (`user_code`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='User recently used apps'


CREATE
DATABASE IF NOT EXISTS EXAMPLE_1;
Expand Down
Loading
Loading