diff --git a/README.md b/README.md index d0f361efe..c7c5521bb 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,8 @@ the following libraries. - new[Conversation with Customized Tools](https://github.com/modelscope/agentscope/blob/main/examples/conversation_with_customized_services/) - new[Mixture of Agents Algorithm](https://github.com/modelscope/agentscope/blob/main/examples/conversation_mixture_of_agents/) - new[Conversation in Stream Mode](https://github.com/modelscope/agentscope/blob/main/examples/conversation_in_stream_mode/) + - new[Conversation with CodeAct Agent](https://github.com/modelscope/agentscope/blob/main/examples/conversation_with_codeact_agent/) + - Game - [Gomoku](https://github.com/modelscope/agentscope/blob/main/examples/game_gomoku) diff --git a/README_ZH.md b/README_ZH.md index a82e53e12..8c89ff927 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -153,10 +153,14 @@ AgentScope支持使用以下库快速部署本地模型服务。 - [通过对话查询SQL信息](./examples/conversation_nl2sql/) - [与RAG智能体对话](./examples/conversation_with_RAG_agents) - new[与gpt-4o模型对话](./examples/conversation_with_gpt-4o) + - new[与自定义服务对话](./examples/conversation_with_customized_services/) + - new[与SoftWare Engineering智能体对话](./examples/conversation_with_swe-agent/) - new[自定义工具函数](./examples/conversation_with_customized_services/) - new[Mixture of Agents算法](https://github.com/modelscope/agentscope/blob/main/examples/conversation_mixture_of_agents/) - new[流式对话](https://github.com/modelscope/agentscope/blob/main/examples/conversation_in_stream_mode/) + - new[与CodeAct智能体对话](https://github.com/modelscope/agentscope/blob/main/examples/conversation_with_codeact_agent/) + - 游戏 - [五子棋](./examples/game_gomoku) diff --git a/docs/sphinx_doc/en/source/tutorial/204-service.md b/docs/sphinx_doc/en/source/tutorial/204-service.md index 38b4047b0..0cfaec6a3 100644 --- a/docs/sphinx_doc/en/source/tutorial/204-service.md +++ b/docs/sphinx_doc/en/source/tutorial/204-service.md @@ -15,6 +15,7 @@ The following table outlines the various Service functions by type. These functi | Service Scene | Service Function Name | Description | |-----------------------------|----------------------------|----------------------------------------------------------------------------------------------------------------| | Code | `execute_python_code` | Execute a piece of Python code, optionally inside a Docker container. | +| | `NoteBookExecutor.run_code_on_notebook` | Compute Execute a segment of Python code in the IPython environment of the NoteBookExecutor, adhering to the IPython interactive computing style. | | Retrieval | `retrieve_from_list` | Retrieve a specific item from a list based on given criteria. | | | `cos_sim` | Compute the cosine similarity between two different embeddings. | | SQL Query | `query_mysql` | Execute SQL queries on a MySQL database and return results. | diff --git a/docs/sphinx_doc/zh_CN/source/tutorial/204-service.md b/docs/sphinx_doc/zh_CN/source/tutorial/204-service.md index 23c145a05..00de68001 100644 --- a/docs/sphinx_doc/zh_CN/source/tutorial/204-service.md +++ b/docs/sphinx_doc/zh_CN/source/tutorial/204-service.md @@ -12,6 +12,7 @@ | Service场景 | Service函数名称 | 描述 | |------------|-----------------------|-----------------------------------------| | 代码 | `execute_python_code` | 执行一段 Python 代码，可选择在 Docker
容器内部执行。 | +| | `NoteBookExecutor.run_code_on_notebook` | 在 NoteBookExecutor 的 IPython 环境中执行一段 Python 代码，遵循 IPython 交互式计算风格。 | | 检索 | `retrieve_from_list` | 根据给定的标准从列表中检索特定项目。 | | | `cos_sim` | 计算2个embedding的余弦相似度。 | | SQL查询 | `query_mysql` | 在 MySQL 数据库上执行 SQL 查询并返回结果。 | diff --git a/examples/conversation_with_codeact_agent/codeact_agent.py b/examples/conversation_with_codeact_agent/codeact_agent.py new file mode 100644 index 000000000..d09b7fda4 --- /dev/null +++ b/examples/conversation_with_codeact_agent/codeact_agent.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# pylint: disable=C0301 +"""An agent class that implements the CodeAct agent. +This agent can execute code interactively as actions. +More details can be found at the paper of codeact agent +https://arxiv.org/abs/2402.01030 +and the original repo of codeact https://github.com/xingyaoww/code-act +""" +from agentscope.agents import AgentBase +from agentscope.message import Msg +from agentscope.service import ( + ServiceResponse, + ServiceExecStatus, + NoteBookExecutor, +) +from agentscope.parsers import RegexTaggedContentParser + +SYSTEM_MESSAGE = """ +You are a helpful assistant that gives helpful, detailed, and polite answers to the user's questions. +You should interact with the interactive Python (Jupyter Notebook) environment and receive the corresponding output when needed. The code written by assistant should be enclosed using tag, for example: print('Hello World!') . +You should attempt fewer things at a time instead of putting too much code in one block. You can install packages through PIP by !pip install [package needed] and should always import packages and define variables before starting to use them. +You should stop and provide an answer when they have already obtained the answer from the execution result. Whenever possible, execute the code for the user using instead of providing it. +Your response should be concise, but do express their thoughts. Always write the code in block to execute them. +You should not ask for the user's input unless necessary. Solve the task on your own and leave no unanswered questions behind. +You should do every thing by your self. +""" # noqa + +EXAMPLE_MESSAGE = """ +Additionally, you are provided with the following code available: +{example_code} +The above code is already available in your interactive Python (Jupyter Notebook) environment, allowing you to directly use these variables and functions without needing to redeclare them. +""" # noqa + + +class CodeActAgent(AgentBase): + """ + The implementation of CodeAct-agent. + The agent can execute code interactively as actions. + More details can be found at the paper of codeact agent + https://arxiv.org/abs/2402.01030 + and the original repo of codeact https://github.com/xingyaoww/code-act + """ + + def __init__( + self, + name: str, + model_config_name: str, + example_code: str = "", + ) -> None: + """ + Initialize the CodeActAgent. + Args: + name(`str`): + The name of the agent. + model_config_name(`str`): + The name of the model configuration. + example_code(Optional`str`): + The example code to be executed bewfore the interaction. + You can import reference libs, define variables + and functions to be called. For example: + + ```python + from agentscope.service import bing_search + import os + + api_key = "{YOUR_BING_API_KEY}" + + def search(question: str): + return bing_search(question, api_key=api_key, num_results=3).content + ``` + + """ # noqa + super().__init__( + name=name, + model_config_name=model_config_name, + ) + self.n_max_executions = 5 + self.example_code = example_code + self.code_executor = NoteBookExecutor() + + sys_msg = Msg(name="system", role="system", content=SYSTEM_MESSAGE) + example_msg = Msg( + name="user", + role="user", + content=EXAMPLE_MESSAGE.format(example_code=self.example_code), + ) + + self.memory.add(sys_msg) + + if self.example_code != "": + code_execution_result = self.code_executor.run_code_on_notebook( + self.example_code, + ) + code_exec_msg = self.handle_code_result( + code_execution_result, + "Example Code excuted: ", + ) + self.memory.add(example_msg) + self.memory.add(code_exec_msg) + self.speak(code_exec_msg) + + self.parser = RegexTaggedContentParser(try_parse_json=False) + + def handle_code_result( + self, + code_execution_result: ServiceResponse, + content_pre_sring: str = "", + ) -> Msg: + """return the message from code result""" + code_exec_content = content_pre_sring + if code_execution_result.status == ServiceExecStatus.SUCCESS: + code_exec_content += "Excution Successful:\n" + else: + code_exec_content += "Excution Failed:\n" + code_exec_content += "Execution Output:\n" + str( + code_execution_result.content, + ) + return Msg(name="user", role="user", content=code_exec_content) + + def reply(self, x: Msg = None) -> Msg: + """The reply function that implements the codeact agent.""" + + self.memory.add(x) + + excution_count = 0 + while ( + self.memory.get_memory(1)[-1].role == "user" + and excution_count < self.n_max_executions + ): + prompt = self.model.format(self.memory.get_memory()) + model_res = self.model(prompt) + msg_res = Msg( + name=self.name, + content=model_res.text, + role="assistant", + ) + self.memory.add(msg_res) + self.speak(msg_res) + res = self.parser.parse(model_res) + code = res.parsed.get("execute") + if code is not None: + code = code.strip() + code_execution_result = ( + self.code_executor.run_code_on_notebook(code) + ) + excution_count += 1 + code_exec_msg = self.handle_code_result(code_execution_result) + self.memory.add(code_exec_msg) + self.speak(code_exec_msg) + + if excution_count == self.n_max_executions: + assert self.memory.get_memory(1)[-1].role == "user" + code_max_exec_msg = Msg( + name="assitant", + role="assistant", + content=( + "I have reached the maximum number " + f"of executions ({self.n_max_executions=}). " + "Can you assist me or ask me another question?" + ), + ) + self.memory.add(code_max_exec_msg) + self.speak(code_max_exec_msg) + return code_max_exec_msg + + return msg_res diff --git a/examples/conversation_with_codeact_agent/codeact_agent_example_modeling.ipynb b/examples/conversation_with_codeact_agent/codeact_agent_example_modeling.ipynb new file mode 100644 index 000000000..f7b80eba2 --- /dev/null +++ b/examples/conversation_with_codeact_agent/codeact_agent_example_modeling.ipynb @@ -0,0 +1,201 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conversation with CodeAct Agent\n", + "\n", + "CodeAct agent is an agent that not only chats but also writes and executes Python code for you.\n", + "More details can be found in the project's related [github repo](https://github.com/xingyaoww/code-act). In the example here, we implement the CodeAct agent, and provide a simple example of how to use the CodeAct agent.\n", + "\n", + "## Prerequisites\n", + "\n", + "- Follow [READMD.md](https://github.com/modelscope/agentscope) to install AgentScope. We require the lastest version, so you should build from source by running `pip install -e .` instead of intalling from pypi. \n", + "- Prepare a model configuration. AgentScope supports both local deployed model services (CPU or GPU) and third-party services. More details and example model configurations please refer to our [tutorial](https://modelscope.github.io/agentscope/en/tutorial/203-model.html).\n", + "\n", + "## Note\n", + "- The example is tested with the following models. For other models, you may need to adjust the prompt.\n", + " - qwen-max" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "YOUR_MODEL_CONFIGURATION_NAME = \"{YOUR_MODEL_CONFIGURATION_NAME}\"\n", + "\n", + "YOUR_MODEL_CONFIGURATION = {\n", + " \"model_type\": \"xxx\", \n", + " \"config_name\": YOUR_MODEL_CONFIGURATION_NAME\n", + " \n", + " # ...\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 1: Initalize the CodeAct-agent\n", + "\n", + "Here we load the CodeAct agent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from codeact_agent import CodeActAgent\n", + "\n", + "import agentscope\n", + "\n", + "agentscope.init(model_configs=YOUR_MODEL_CONFIGURATION)\n", + "\n", + "import nest_asyncio\n", + "nest_asyncio.apply()\n", + "agent = CodeActAgent(\n", + " name=\"assistant\",\n", + " model_config_name=YOUR_MODEL_CONFIGURATION_NAME,\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "###Step 2: Ask the CodeAct-agent to execute tasks\n", + "\n", + "Here, we ask the CodeAct-agent to implement a statistical simulation and modeling procedure." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-05-16 19:48:01.620 | INFO | agentscope.models.model:__init__:201 - Initialize model by configuration [dashscope_chat-qwen-max]\n", + "2024-05-16 19:48:01.624 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.call_counter] to SqliteMonitor with unit [times] and quota [None]\n", + "2024-05-16 19:48:01.627 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.prompt_tokens] to SqliteMonitor with unit [token] and quota [None]\n", + "2024-05-16 19:48:01.630 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.completion_tokens] to SqliteMonitor with unit [token] and quota [None]\n", + "2024-05-16 19:48:01.632 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.total_tokens] to SqliteMonitor with unit [token] and quota [None]\n", + "user: Given y = 0.9x + 6.1, randomly sample data points as pairs of (x, y). Then fit a linear regression on the sampled data and plot the points, fitted line, and ground-truth line.\n", + "assistant: To accomplish this task, I will follow these steps:\n", + "\n", + "1. Generate random `x` values within a reasonable range.\n", + "2. Use the equation `y = 0.9x + 6.1` to compute the corresponding `y` values for each `x`.\n", + "3. Add some noise to the `y` values to simulate real-world data variability.\n", + "4. Split the data into training and testing sets (though for simplicity and since you didn't specify, we'll just use all the data for fitting).\n", + "5. Fit a linear regression model on the generated data.\n", + "6. Plot the original data points, the fitted line from the regression, and the ground-truth line.\n", + "\n", + "Let's start by generating the data points and fitting the regression model.\n", + "\n", + "[execute]\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "# Set a seed for reproducibility\n", + "np.random.seed(42)\n", + "\n", + "# Generate random x values between 0 and 100\n", + "x_values = np.random.rand(100) * 100\n", + "\n", + "# Compute the corresponding y values without noise\n", + "y_values_ground_truth = 0.9 * x_values + 6.1\n", + "\n", + "# Add some noise to y values\n", + "noise = np.random.normal(0, 10, size=x_values.shape)\n", + "y_values_noisy = y_values_ground_truth + noise\n", + "\n", + "# Combine into a single dataset\n", + "data = np.column_stack((x_values, y_values_noisy))\n", + "\n", + "# Fit a linear regression model\n", + "model = LinearRegression()\n", + "model.fit(data[:, 0].reshape(-1, 1), data[:, 1])\n", + "\n", + "# Prepare to plot\n", + "plt.figure(figsize=(10, 6))\n", + "[/execute]\n", + "user: Execution Output:\n", + "['
', 'Unsupported display type']\n", + "assistant: It seems there was an issue with displaying the plot directly here. No worries, I will still generate the plot and save it to a file instead, which you can view separately. Let me adjust the code to save the plot as an image file.\n", + "\n", + "[execute]\n", + "# Plotting the data points, fitted line, and ground-truth line\n", + "plt.scatter(data[:, 0], data[:, 1], label='Sampled Data', color='blue')\n", + "plt.plot(data[:, 0], model.predict(data[:, 0].reshape(-1, 1)), label='Fitted Line', color='red', linestyle='--')\n", + "plt.plot(data[:, 0], y_values_ground_truth, label='Ground-Truth Line', color='green')\n", + "\n", + "plt.title('Linear Regression on Sampled Data')\n", + "plt.xlabel('X')\n", + "plt.ylabel('Y')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "\n", + "# Save the plot to a file\n", + "plt.savefig('linear_regression_plot.png')\n", + "plt.close()\n", + "[/execute]\n", + "\n", + "The plot has been saved as 'linear_regression_plot.png'. You can view this file to see the sampled data points, the fitted line from the linear regression, and the ground-truth line based on the equation `y = 0.9x + 6.1`. If you need further analysis or have any other requests, feel free to ask!\n", + "user: Execution Output:\n", + "[]\n", + "assistant: It appears the output confirmation was suppressed in this environment, but typically, when running the code locally or in a supported environment, you would see a message indicating the plot was successfully saved to 'linear_regression_plot.png'.\n", + "\n", + "Since we cannot directly view the saved file here, trust that the file has been created with the following components:\n", + "\n", + "- **Sampled Data Points**: Represented as blue dots, scattered according to the generated `x` values and the noisy `y` values.\n", + "- **Fitted Line**: Shown as a red dashed line, representing the linear regression model's prediction based on the sampled data.\n", + "- **Ground-Truth Line**: Displayed as a green line, illustrating the true relationship defined by `y = 0.9x + 6.1`.\n", + "\n", + "If you need further assistance or another operation, such as analyzing the quality of the fit or re-running the process with different parameters, please let me know!\n" + ] + } + ], + "source": [ + "from loguru import logger\n", + "from agentscope.message import Msg\n", + "\n", + "mss = Msg(\n", + " name=\"user\", \n", + " content=\"Given y = 0.9x + 6.1, randomly sample data points as pairs of (x, y). Then fit a linear regression on the sampled data and plot the points, fitted line, and ground-truth line.\", \n", + " role=\"user\"\n", + ")\n", + "logger.chat(mss)\n", + "answer_mss1 = agent(mss)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "datajuicer", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/conversation_with_codeact_agent/codeact_agent_example_tools.ipynb b/examples/conversation_with_codeact_agent/codeact_agent_example_tools.ipynb new file mode 100644 index 000000000..787e1a15f --- /dev/null +++ b/examples/conversation_with_codeact_agent/codeact_agent_example_tools.ipynb @@ -0,0 +1,189 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conversation with CodeAct Agent\n", + "\n", + "CodeAct agent is an agent that not only chats but also writes and executes Python code for you.\n", + "More details can be found in the project's related [github repo](https://github.com/xingyaoww/code-act). \n", + "\n", + "In the following CodeAct agent example, we demonstrate another method of empowering the agent with the capability to invoke tools, specifically by directly providing the agent with the corresponding code for the tools and then allowing the agent to utilize them independently.\n", + "Interm of tool usage, ReAct agent also enables the agent to use tools to solve problems, but in a different way. You can refer to the [ReAct agent](https://github.com/modelscope/agentscope/blob/main/examples/conversation_with_react_agent/main.ipynb) example for more details and compare the two.\n", + "\n", + "## Prerequisites\n", + "\n", + "- Follow [READMD.md](https://github.com/modelscope/agentscope) to install AgentScope. We require the lastest version, so you should build from source by running `pip install -e .` instead of intalling from pypi. \n", + "- Prepare a model configuration. AgentScope supports both local deployed model services (CPU or GPU) and third-party services. More details and example model configurations please refer to our [tutorial](https://modelscope.github.io/agentscope/en/tutorial/203-model.html).\n", + "\n", + "## Note\n", + "- The example is tested with the following models. For other models, you may need to adjust the prompt.\n", + " - qwen-max" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "YOUR_MODEL_CONFIGURATION_NAME = \"{YOUR_MODEL_CONFIGURATION_NAME}\"\n", + "\n", + "YOUR_MODEL_CONFIGURATION = {\n", + " \"model_type\": \"xxx\", \n", + " \"config_name\": YOUR_MODEL_CONFIGURATION_NAME\n", + " \n", + " # ...\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 1: Initalize the AgentScope environment\n", + "\n", + "Here we initialize the AgentScope environment by calling the `agentscope.init` function. The `model_configs` parameter specifies the path to the model configuration file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import agentscope\n", + "\n", + "agentscope.init(model_configs=YOUR_MODEL_CONFIGURATION)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2: Set up the code for tool usage, and init the CodeAct Agent.\n", + "\n", + "Here, we provide the CodeAct agent with the interactive code that the agent can use during the conversation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from loguru import logger\n", + "from codeact_agent import CodeActAgent\n", + "\n", + "from agentscope.message import Msg\n", + "\n", + "YOUR_BING_API_KEY = \"xxx\" # fill in your bing api key here\n", + "\n", + "example_code = f\"\"\"\n", + "from agentscope.service import bing_search\n", + "import os\n", + "\n", + "api_key = \"{YOUR_BING_API_KEY}\"\n", + "\n", + "def search(question: str):\n", + " return bing_search(question, api_key=api_key, num_results=3).content\n", + "\"\"\"\n", + "\n", + "import nest_asyncio\n", + "nest_asyncio.apply()\n", + "agent = CodeActAgent(\n", + " name=\"assistant\",\n", + " model_config_name=YOUR_MODEL_CONFIGURATION_NAME,\n", + " example_code=example_code\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3: Ask the CodeAct-agent to execute tasks\n", + "\n", + "Here, we ask the CodeAct-agent with the example question in the ReAct Paper: `\"Aside from the Apple Remote, what other device can control the program Apple Remote was originally designed to interact with?\"` as follows, same as in the [ReAct-agent]((https://github.com/modelscope/agentscope/blob/main/examples/conversation_with_react_agent/main.ipynb)) example." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-05-17 10:58:04.214 | INFO | agentscope.models.model:__init__:201 - Initialize model by configuration [dashscope_chat-qwen-max]\n", + "2024-05-17 10:58:04.215 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.call_counter] to SqliteMonitor with unit [times] and quota [None]\n", + "2024-05-17 10:58:04.218 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.prompt_tokens] to SqliteMonitor with unit [token] and quota [None]\n", + "2024-05-17 10:58:04.220 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.completion_tokens] to SqliteMonitor with unit [token] and quota [None]\n", + "2024-05-17 10:58:04.222 | INFO | agentscope.utils.monitor:register:417 - Register metric [qwen-max.total_tokens] to SqliteMonitor with unit [token] and quota [None]\n", + "user: Example Code excuted: Excution Successful:\n", + " Execution Output:\n", + "[]\n", + "user: Aside from the Apple Remote, what other device can control the program Apple Remote was originally designed to interact with?\n", + "assistant: [execute]\n", + "search_result = search(\"Aside from the Apple Remote, what other device can control the program Apple Remote was originally designed to interact with?\")\n", + "search_result\n", + "[/execute]\n", + "user: Excution Successful:\n", + " Execution Output:\n", + "[\"[{'title': 'Multi-Agent实践第4期：智能体的“想”与“做”-ReAct Agent - 知乎',\\n 'link': 'https://zhuanlan.zhihu.com/p/689675968',\\n 'snippet': 'Other than the Apple Remote, you can use a supported TV or receiver remote, a network-based remote for home-control systems, an infrared remote (commonly known as a universal remote), or other Apple devices like an iPhone or iPad to control the Apple'},\\n {'title': 'REACT：在语言模型中协同推理与行动，使其能够解决各种 ...',\\n 'link': 'https://blog.csdn.net/fogdragon/article/details/132550968',\\n 'snippet': 'Q：Aside from the Apple Remote, what other device can control the program Apple Remote was originally designed to intect with? 除了苹果遥控器之外，还有哪些设备可以控制最初设计用于连接的苹果遥控器所配合的程序？'},\\n {'title': '《ReAct: SYNERGIZING REASONING AND ACTING IN ...',\\n 'link': 'https://www.cnblogs.com/LittleHann/p/17541295.html',\\n 'snippet': 'Aside from the Apple Remote, what other devices can control the program Apple Remote was originally designed to interact with?'}]\"]\n", + "assistant: According to the search results, aside from the Apple Remote, you can use the following devices to control the program it was designed to interact with:\n", + "\n", + "1. A supported TV or receiver remote.\n", + "2. A network-based remote for home-control systems.\n", + "3. An infrared remote (universal remote).\n", + "4. Other Apple devices such as an iPhone or iPad.\n", + "\n", + "These alternatives allow for similar control functionality as the Apple Remote.\n" + ] + } + ], + "source": [ + "mss = Msg(\n", + " name=\"user\", \n", + " content=\"Aside from the Apple Remote, what other device can control the program Apple Remote was originally designed to interact with?\", \n", + " role=\"user\"\n", + ")\n", + "logger.chat(mss)\n", + "answer_mss1 = agent(mss)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The CodeAct agent successfully use the given search tool functions and return the answer accordingly. This demonstrates the wide range of usage of agent with coding abilities.\n", + "Feel free to explore the Agent by yourself!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "datajuicer", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/conversation_with_codeact_agent/linear_regression_plot.png b/examples/conversation_with_codeact_agent/linear_regression_plot.png new file mode 100644 index 000000000..a70267ed2 Binary files /dev/null and b/examples/conversation_with_codeact_agent/linear_regression_plot.png differ diff --git a/setup.py b/setup.py index ac1fa06e0..812b9436d 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,9 @@ "google-generativeai>=0.4.0", "zhipuai", "litellm", + "notebook", + "nbclient", + "nbformat", "psutil", "scipy", ] diff --git a/src/agentscope/file_manager.py b/src/agentscope/file_manager.py index ab042f5b5..4adde73a4 100644 --- a/src/agentscope/file_manager.py +++ b/src/agentscope/file_manager.py @@ -2,6 +2,7 @@ """Manage the file system for saving files, code and logs.""" import json import os +import io from typing import Any, Union, Optional, List, Literal from pathlib import Path import numpy as np @@ -169,7 +170,7 @@ def save_api_invocation( def save_image( self, - image: Union[str, np.ndarray], + image: Union[str, np.ndarray, bytes], filename: Optional[str] = None, ) -> str: """Save image file locally, and return the local image path. @@ -195,11 +196,21 @@ def save_image( if isinstance(image, str): # download the image from url _download_file(image, path_file) - else: + elif isinstance(image, np.ndarray): from PIL import Image # save image via PIL Image.fromarray(image).save(path_file) + elif isinstance(image, bytes): + from PIL import Image + + # save image via bytes + Image.open(io.BytesIO(image)).save(path_file) + else: + raise ValueError( + f"Unsupported image type: {type(image)}" + "Must be str, np.ndarray, or bytes.", + ) return path_file diff --git a/src/agentscope/service/__init__.py b/src/agentscope/service/__init__.py index 2a0ba3e53..b7a2471aa 100644 --- a/src/agentscope/service/__init__.py +++ b/src/agentscope/service/__init__.py @@ -4,6 +4,7 @@ from .execute_code.exec_python import execute_python_code from .execute_code.exec_shell import execute_shell_command +from .execute_code.exec_notebook import NoteBookExecutor from .file.common import ( create_file, delete_file, @@ -92,6 +93,7 @@ def get_help() -> None: "dblp_search_publications", "dblp_search_authors", "dblp_search_venues", + "NoteBookExecutor", "dashscope_image_to_text", "dashscope_text_to_image", "dashscope_text_to_audio", diff --git a/src/agentscope/service/execute_code/exec_notebook.py b/src/agentscope/service/execute_code/exec_notebook.py new file mode 100644 index 000000000..784cf41f9 --- /dev/null +++ b/src/agentscope/service/execute_code/exec_notebook.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# pylint: disable=C0301 +"""Service for executing jupyter notebooks interactively +Partially referenced the implementation of https://github.com/geekan/MetaGPT/blob/main/metagpt/actions/di/execute_nb_code.py +""" # noqa +import base64 +import uuid +import asyncio +from loguru import logger + + +try: + from nbclient import NotebookClient + from nbclient.exceptions import CellTimeoutError, DeadKernelError + import nbformat +except ImportError as import_error: + from agentscope.utils.tools import ImportErrorReporter + + nbclient = ImportErrorReporter(import_error) + nbformat = ImportErrorReporter(import_error) + NotebookClient = ImportErrorReporter(import_error) + +from agentscope.service.service_status import ServiceExecStatus +from agentscope.service.service_response import ServiceResponse +from agentscope.file_manager import file_manager + + +class NoteBookExecutor: + """ + Class for executing jupyter notebooks block interactively. + To use the service function, you should first init the class, then call the + run_code_on_notebook function. + + Example: + + ```ipython + from agentscope.service.service_toolkit import * + from agentscope.service.execute_code.exec_notebook import * + nbe = NoteBookExecutor() + code = "print('helloworld')" + # calling directly + nbe.run_code_on_notebook(code) + + >>> Executing function run_code_on_notebook with arguments: + >>> code: print('helloworld') + >>> END + + # calling with service toolkit + service_toolkit = ServiceToolkit() + service_toolkit.add(nbe.run_code_on_notebook) + input_obs = [{"name": "run_code_on_notebook", "arguments":{"code": code}}] + res_of_string_input = service_toolkit.parse_and_call_func(input_obs) + + "1. Execute function run_code_on_notebook\n [ARGUMENTS]:\n code: print('helloworld')\n [STATUS]: SUCCESS\n [RESULT]: ['helloworld\\n']\n" + + ``` + """ # noqa + + def __init__( + self, + timeout: int = 300, + ) -> None: + """ + The construct function of the NoteBookExecutor. + Args: + timeout (Optional`int`): + The timeout for each cell execution. + Default to 300. + """ + self.nb = nbformat.v4.new_notebook() + self.nb_client = NotebookClient(nb=self.nb) + self.timeout = timeout + + asyncio.run(self._start_client()) + + def _output_parser(self, output: dict) -> str: + """Parse the output of the notebook cell and return str""" + if output["output_type"] == "stream": + return output["text"] + elif output["output_type"] == "execute_result": + return output["data"]["text/plain"] + elif output["output_type"] == "display_data": + if "image/png" in output["data"]: + file_path = self._save_image(output["data"]["image/png"]) + return f"Displayed image saved to {file_path}" + else: + return "Unsupported display type" + elif output["output_type"] == "error": + return output["traceback"] + else: + logger.info(f"Unsupported output encountered: {output}") + return "Unsupported output encountered" + + async def _start_client(self) -> None: + """start notebook client""" + if self.nb_client.kc is None or not await self.nb_client.kc.is_alive(): + self.nb_client.create_kernel_manager() + self.nb_client.start_new_kernel() + self.nb_client.start_new_kernel_client() + + async def _kill_client(self) -> None: + """kill notebook client""" + if ( + self.nb_client.km is not None + and await self.nb_client.km.is_alive() + ): + await self.nb_client.km.shutdown_kernel(now=True) + await self.nb_client.km.cleanup_resources() + + self.nb_client.kc.stop_channels() + self.nb_client.kc = None + self.nb_client.km = None + + async def _restart_client(self) -> None: + """Restart the notebook client""" + await self._kill_client() + self.nb_client = NotebookClient(self.nb, timeout=self.timeout) + await self._start_client() + + async def _run_cell(self, cell_index: int) -> ServiceResponse: + """Run a cell in the notebook by its index""" + try: + self.nb_client.execute_cell(self.nb.cells[cell_index], cell_index) + return ServiceResponse( + status=ServiceExecStatus.SUCCESS, + content=[ + self._output_parser(output) + for output in self.nb.cells[cell_index].outputs + ], + ) + except DeadKernelError: + await self.reset() + return ServiceResponse( + status=ServiceExecStatus.ERROR, + content="DeadKernelError when executing cell, reset kernel", + ) + except CellTimeoutError: + assert self.nb_client.km is not None + await self.nb_client.km.interrupt_kernel() + return ServiceResponse( + status=ServiceExecStatus.ERROR, + content=( + "CellTimeoutError when executing cell" + ", code execution timeout" + ), + ) + except Exception as e: + return ServiceResponse( + status=ServiceExecStatus.ERROR, + content=str(e), + ) + + @property + def cells_length(self) -> int: + """return cell length""" + return len(self.nb.cells) + + async def async_run_code_on_notebook(self, code: str) -> ServiceResponse: + """ + Run the code on interactive notebook + """ + self.nb.cells.append(nbformat.v4.new_code_cell(code)) + cell_index = self.cells_length - 1 + return await self._run_cell(cell_index) + + def run_code_on_notebook(self, code: str) -> ServiceResponse: + """ + Run the code on interactive jupyter notebook. + + Args: + code (`str`): + The Python code to be executed in the interactive notebook. + + Returns: + `ServiceResponse`: whether the code execution was successful, + and the output of the code execution. + """ + return asyncio.run(self.async_run_code_on_notebook(code)) + + def reset_notebook(self) -> ServiceResponse: + """ + Reset the notebook + """ + asyncio.run(self._restart_client()) + return ServiceResponse( + status=ServiceExecStatus.SUCCESS, + content="Reset notebook", + ) + + def _save_image(self, image_base64: str) -> str: + """Save image data to a file. + The image name is generated randomly here""" + image_data = base64.b64decode(image_base64) + filename = f"display_image_{uuid.uuid4().hex}.png" + return file_manager.save_image(image_data, filename) diff --git a/src/agentscope/service/service_toolkit.py b/src/agentscope/service/service_toolkit.py index 299b23d3f..afa112bd9 100644 --- a/src/agentscope/service/service_toolkit.py +++ b/src/agentscope/service/service_toolkit.py @@ -496,7 +496,8 @@ def bing_search(query: str, api_key: str, num_results: int=10): ) # The arguments that requires the agent to specify - args_agent = set(argsspec.args) - set(kwargs.keys()) + # to support class method, the self args are deprecated + args_agent = set(argsspec.args) - set(kwargs.keys()) - {"self", "cls"} # Check if the arguments from agent have descriptions in docstring args_description = { @@ -653,7 +654,8 @@ def bing_search(query: str, api_key: str, num_results: int=10): ) # The arguments that requires the agent to specify - args_agent = set(argsspec.args) - set(kwargs.keys()) + # we remove the self argument, for class methods + args_agent = set(argsspec.args) - set(kwargs.keys()) - {"self", "cls"} # Check if the arguments from agent have descriptions in docstring args_description = { diff --git a/tests/execute_notebook_code_test.py b/tests/execute_notebook_code_test.py new file mode 100644 index 000000000..0259d8f76 --- /dev/null +++ b/tests/execute_notebook_code_test.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +""" iPython code execution test.""" +import unittest +from agentscope.service.execute_code.exec_notebook import NoteBookExecutor +from agentscope.service.service_status import ServiceExecStatus + + +class ExecuteNotebookCodeTest(unittest.TestCase): + """ + Notebook code execution test. + """ + + def setUp(self) -> None: + """Init for ExecuteNotebookCodeTest.""" + self.executor = NoteBookExecutor() + + # Basic expression + self.arg0 = {"code": "print('Hello World')"} + # Using External Libraries + self.arg1 = {"code": "import math\nprint(math.sqrt(16))"} + # No input code + self.arg2 = {"code": ""} + # test without print + self.arg3 = {"code": "1+1"} + + def test_basic_expression(self) -> None: + """Execute basic expression test.""" + response = self.executor.run_code_on_notebook(self.arg0["code"]) + self.assertEqual(response.status, ServiceExecStatus.SUCCESS) + self.assertIn("Hello World\n", response.content[0]) + + def test_using_external_libs(self) -> None: + """Execute using external libs test.""" + response = self.executor.run_code_on_notebook(self.arg1["code"]) + self.assertEqual(response.status, ServiceExecStatus.SUCCESS) + self.assertIn("4.0\n", response.content[0]) + + def test_no_input_code(self) -> None: + """Execute no input code test.""" + response = self.executor.run_code_on_notebook(self.arg2["code"]) + self.assertEqual(response.status, ServiceExecStatus.SUCCESS) + self.assertEqual(response.content, []) + + def test_no_print(self) -> None: + """Execute no print test.""" + response = self.executor.run_code_on_notebook(self.arg3["code"]) + self.assertEqual(response.status, ServiceExecStatus.SUCCESS) + self.assertIn("2", response.content[0]) + + +if __name__ == "__main__": + unittest.main()