From 7f656c300fbd7dab23533acbbfa1b245eaff8953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 23 Oct 2023 11:21:16 +0800 Subject: [PATCH 001/232] fixbug: issues#445 --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 562a653f3..cff7c3c0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,4 +48,3 @@ websocket-client==0.58.0 aiofiles~=23.2.1 pygments~=2.16.1 requests~=2.31.0 -yaml~=0.2.5 \ No newline at end of file From 5e8ada5cfffd470a7513630391077f0f291e8f8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 31 Oct 2023 15:23:37 +0800 Subject: [PATCH 002/232] refactor: Message --- metagpt/schema.py | 106 +++++++++++++++++++++++++--------- tests/metagpt/test_message.py | 22 ++++--- 2 files changed, 92 insertions(+), 36 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index bdca093c2..1124fb28e 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -4,13 +4,15 @@ @Time : 2023/5/8 22:12 @Author : alexanderwu @File : schema.py +@Modified By: mashenquan, 2023-10-31, optimize class members. """ from __future__ import annotations -from dataclasses import dataclass, field -from typing import Type, TypedDict +import json +from json import JSONDecodeError +from typing import Dict, List, TypedDict -from pydantic import BaseModel +from pydantic import BaseModel, Field from metagpt.logs import logger @@ -20,16 +22,44 @@ class RawMessage(TypedDict): role: str -@dataclass -class Message: +class Message(BaseModel): """list[: ]""" + content: str - instruct_content: BaseModel = field(default=None) - role: str = field(default='user') # system / user / assistant - cause_by: Type["Action"] = field(default="") - sent_from: str = field(default="") - send_to: str = field(default="") - restricted_to: str = field(default="") + instruct_content: BaseModel = None + meta_info: Dict = Field(default_factory=dict) + route: List[Dict] = Field(default_factory=list) + + def __init__(self, content, **kwargs): + super(Message, self).__init__( + content=content or kwargs.get("content"), + instruct_content=kwargs.get("instruct_content"), + meta_info=kwargs.get("meta_info", {}), + route=kwargs.get("route", []), + ) + + attribute_names = Message.__annotations__.keys() + for k, v in kwargs.items(): + if k in attribute_names: + continue + self.meta_info[k] = v + + def get_meta(self, key): + return self.meta_info.get(key) + + def set_meta(self, key, value): + self.meta_info[key] = value + + @property + def role(self): + return self.get_meta("role") + + @property + def cause_by(self): + return self.get_meta("cause_by") + + def set_role(self, v): + self.set_meta("role", v) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) @@ -39,45 +69,67 @@ def __repr__(self): return self.__str__() def to_dict(self) -> dict: - return { - "role": self.role, - "content": self.content - } + return {"role": self.role, "content": self.content} + + def save(self) -> str: + return self.json(exclude_none=True) + + @staticmethod + def load(v): + try: + d = json.loads(v) + return Message(**d) + except JSONDecodeError as err: + logger.error(f"parse json failed: {v}, error:{err}") + return None -@dataclass class UserMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'user') + super(Message, self).__init__(content=content, meta_info={"role": "user"}) -@dataclass class SystemMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'system') + super().__init__(content=content, meta_info={"role": "system"}) -@dataclass class AIMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'assistant') + super().__init__(content=content, meta_info={"role": "assistant"}) + +if __name__ == "__main__": + m = Message("a", role="v1") + m.set_role("v2") + v = m.save() + m = Message.load(v) -if __name__ == '__main__': - test_content = 'test_message' + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] logger.info(msgs) + + jsons = [ + UserMessage(test_content).save(), + SystemMessage(test_content).save(), + AIMessage(test_content).save(), + Message(test_content, role="QA").save(), + ] + logger.info(jsons) diff --git a/tests/metagpt/test_message.py b/tests/metagpt/test_message.py index e26f38381..4f46311ce 100644 --- a/tests/metagpt/test_message.py +++ b/tests/metagpt/test_message.py @@ -11,26 +11,30 @@ def test_message(): - msg = Message(role='User', content='WTF') - assert msg.to_dict()['role'] == 'User' - assert 'User' in str(msg) + msg = Message(role="User", content="WTF") + assert msg.to_dict()["role"] == "User" + assert "User" in str(msg) def test_all_messages(): - test_content = 'test_message' + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] for msg in msgs: assert msg.content == test_content def test_raw_message(): - msg = RawMessage(role='user', content='raw') - assert msg['role'] == 'user' - assert msg['content'] == 'raw' + msg = RawMessage(role="user", content="raw") + assert msg["role"] == "user" + assert msg["content"] == "raw" with pytest.raises(KeyError): - assert msg['1'] == 1, "KeyError: '1'" + assert msg["1"] == 1, "KeyError: '1'" + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 545d77ce0deac125c14ff8c902ca49ff5ded8cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 1 Nov 2023 20:08:58 +0800 Subject: [PATCH 003/232] refactor: Refactor Message transmission & filtering --- examples/agent_creator.py | 20 +- examples/build_customized_agent.py | 28 +-- examples/debate.py | 47 +++-- examples/sk_agent.py | 9 +- metagpt/actions/action.py | 3 +- metagpt/actions/write_code.py | 9 +- metagpt/const.py | 5 + metagpt/environment.py | 48 +++-- metagpt/memory/longterm_memory.py | 16 +- metagpt/memory/memory.py | 11 +- metagpt/roles/engineer.py | 68 +++++-- metagpt/roles/qa_engineer.py | 46 +++-- metagpt/roles/researcher.py | 15 +- metagpt/roles/role.py | 165 ++++++++++------ metagpt/roles/seacher.py | 36 ++-- metagpt/roles/sk_agent.py | 6 +- metagpt/schema.py | 193 ++++++++++++++++++- metagpt/software_company.py | 22 ++- metagpt/utils/common.py | 16 +- metagpt/utils/named.py | 21 ++ tests/metagpt/actions/test_write_prd.py | 3 +- tests/metagpt/memory/test_longterm_memory.py | 31 +-- tests/metagpt/memory/test_memory_storage.py | 72 ++++--- tests/metagpt/planner/test_action_planner.py | 7 +- tests/metagpt/planner/test_basic_planner.py | 6 +- tests/metagpt/roles/mock.py | 27 +-- tests/metagpt/roles/test_architect.py | 5 +- tests/metagpt/roles/test_engineer.py | 13 +- tests/metagpt/test_environment.py | 3 +- tests/metagpt/utils/test_serialize.py | 7 +- 30 files changed, 660 insertions(+), 298 deletions(-) create mode 100644 metagpt/utils/named.py diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 325e7c260..d13cbcff2 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -1,22 +1,24 @@ -''' +""" Filename: MetaGPT/examples/agent_creator.py Created Date: Tuesday, September 12th 2023, 3:28:37 pm Author: garylin2099 -''' +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +""" import re -from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT from metagpt.actions import Action +from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger +from metagpt.utils.common import get_object_name with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() -class CreateAgent(Action): +class CreateAgent(Action): PROMPT_TEMPLATE = """ ### BACKGROUND You are using an agent framework called metagpt to write agents capable of different actions, @@ -34,7 +36,6 @@ class CreateAgent(Action): """ async def run(self, example: str, instruction: str): - prompt = self.PROMPT_TEMPLATE.format(example=example, instruction=instruction) # logger.info(prompt) @@ -46,13 +47,14 @@ async def run(self, example: str, instruction: str): @staticmethod def parse_code(rsp): - pattern = r'```python(.*)```' + pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text + class AgentCreator(Role): def __init__( self, @@ -72,15 +74,15 @@ async def _act(self) -> Message: instruction = msg.content code_text = await CreateAgent().run(example=self.agent_template, instruction=instruction) - msg = Message(content=code_text, role=self.profile, cause_by=todo) + msg = Message(content=code_text, role=self.profile, cause_by=get_object_name(todo)) return msg + if __name__ == "__main__": import asyncio async def main(): - agent_template = MULTI_ACTION_AGENT_CODE_EXAMPLE creator = AgentCreator(agent_template=agent_template) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index 87d7a9c76..a953dee15 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -1,21 +1,23 @@ -''' +""" Filename: MetaGPT/examples/build_customized_agent.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +""" +import asyncio import re import subprocess -import asyncio import fire from metagpt.actions import Action +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger +from metagpt.utils.common import get_object_name -class SimpleWriteCode(Action): +class SimpleWriteCode(Action): PROMPT_TEMPLATE = """ Write a python function that can {instruction} and provide two runnnable test cases. Return ```python your_code_here ``` with NO other texts, @@ -35,7 +37,6 @@ def __init__(self, name="SimpleWriteCode", context=None, llm=None): super().__init__(name, context, llm) async def run(self, instruction: str): - prompt = self.PROMPT_TEMPLATE.format(instruction=instruction) rsp = await self._aask(prompt) @@ -46,11 +47,12 @@ async def run(self, instruction: str): @staticmethod def parse_code(rsp): - pattern = r'```python(.*)```' + pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else rsp return code_text + class SimpleRunCode(Action): def __init__(self, name="SimpleRunCode", context=None, llm=None): super().__init__(name, context, llm) @@ -61,6 +63,7 @@ async def run(self, code_text: str): logger.info(f"{code_result=}") return code_result + class SimpleCoder(Role): def __init__( self, @@ -75,14 +78,15 @@ async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo - msg = self._rc.memory.get()[-1] # retrieve the latest memory + msg = self._rc.memory.get()[-1] # retrieve the latest memory instruction = msg.content code_text = await SimpleWriteCode().run(instruction) - msg = Message(content=code_text, role=self.profile, cause_by=todo) + msg = Message(content=code_text, role=self.profile, cause_by=get_object_name(todo)) return msg + class RunnableCoder(Role): def __init__( self, @@ -116,7 +120,7 @@ async def _act(self) -> Message: code_text = msg.content result = await SimpleRunCode().run(code_text) - msg = Message(content=result, role=self.profile, cause_by=todo) + msg = Message(content=result, role=self.profile, cause_by=get_object_name(todo)) self._rc.memory.add(msg) return msg @@ -128,6 +132,7 @@ async def _react(self) -> Message: await self._act() return Message(content="All job done", role=self.profile) + def main(msg="write a function that calculates the sum of a list"): # role = SimpleCoder() role = RunnableCoder() @@ -135,5 +140,6 @@ def main(msg="write a function that calculates the sum of a list"): result = asyncio.run(role.run(msg)) logger.info(result) -if __name__ == '__main__': + +if __name__ == "__main__": fire.Fire(main) diff --git a/examples/debate.py b/examples/debate.py index 05db28070..ade1a6fc4 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -1,17 +1,20 @@ -''' +""" Filename: MetaGPT/examples/debate.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +""" import asyncio import platform + import fire -from metagpt.software_company import SoftwareCompany from metagpt.actions import Action, BossRequirement +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger +from metagpt.software_company import SoftwareCompany + class ShoutOut(Action): """Action: Shout out loudly in a debate (quarrel)""" @@ -31,7 +34,6 @@ def __init__(self, name="ShoutOut", context=None, llm=None): super().__init__(name, context, llm) async def run(self, context: str, name: str, opponent_name: str): - prompt = self.PROMPT_TEMPLATE.format(context=context, name=name, opponent_name=opponent_name) # logger.info(prompt) @@ -39,6 +41,7 @@ async def run(self, context: str, name: str, opponent_name: str): return rsp + class Trump(Role): def __init__( self, @@ -55,13 +58,13 @@ def __init__( async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] + self._rc.news = [msg for msg in self._rc.news if msg.is_recipient({self.name})] return len(self._rc.news) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions([ShoutOut]) + msg_history = self._rc.memory.get_by_actions([ShoutOut.get_class_name()]) context = [] for m in msg_history: context.append(str(m)) @@ -72,13 +75,14 @@ async def _act(self) -> Message: msg = Message( content=rsp, role=self.profile, - cause_by=ShoutOut, - sent_from=self.name, - send_to=self.opponent_name, + cause_by=ShoutOut.get_class_name(), + tx_from=self.name, + tx_to=self.opponent_name, ) return msg + class Biden(Role): def __init__( self, @@ -96,13 +100,14 @@ async def _observe(self) -> int: await super()._observe() # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, # disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.cause_by == BossRequirement or msg.send_to == self.name] + message_filter = {BossRequirement.get_class_name(), self.name} + self._rc.news = [msg for msg in self._rc.news if msg.is_recipient(message_filter)] return len(self._rc.news) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) + msg_history = self._rc.memory.get_by_actions([BossRequirement.get_class_name(), ShoutOut.get_class_name()]) context = [] for m in msg_history: context.append(str(m)) @@ -113,17 +118,19 @@ async def _act(self) -> Message: msg = Message( content=rsp, role=self.profile, - cause_by=ShoutOut, - sent_from=self.name, - send_to=self.opponent_name, + cause_by=ShoutOut.get_class_name(), + tx_from=self.name, + tx_to=self.opponent_name, ) return msg -async def startup(idea: str, investment: float = 3.0, n_round: int = 5, - code_review: bool = False, run_tests: bool = False): + +async def startup( + idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False +): """We reuse the startup paradigm for roles to interact with each other. - Now we run a startup of presidents and watch they quarrel. :) """ + Now we run a startup of presidents and watch they quarrel. :)""" company = SoftwareCompany() company.hire([Biden(), Trump()]) company.invest(investment) @@ -133,7 +140,7 @@ async def startup(idea: str, investment: float = 3.0, n_round: int = 5, def main(idea: str, investment: float = 3.0, n_round: int = 10): """ - :param idea: Debate topic, such as "Topic: The U.S. should commit more in climate change fighting" + :param idea: Debate topic, such as "Topic: The U.S. should commit more in climate change fighting" or "Trump: Climate change is a hoax" :param investment: contribute a certain dollar amount to watch the debate :param n_round: maximum rounds of the debate @@ -144,5 +151,5 @@ def main(idea: str, investment: float = 3.0, n_round: int = 10): asyncio.run(startup(idea, investment, n_round)) -if __name__ == '__main__': +if __name__ == "__main__": fire.Fire(main) diff --git a/examples/sk_agent.py b/examples/sk_agent.py index a7513e838..19ee53669 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -4,6 +4,7 @@ @Time : 2023/9/13 12:36 @Author : femto Zheng @File : sk_agent.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. """ import asyncio @@ -39,7 +40,7 @@ async def basic_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) async def sequential_planner_example(): @@ -53,7 +54,7 @@ async def sequential_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) async def basic_planner_web_search_example(): @@ -64,7 +65,7 @@ async def basic_planner_web_search_example(): role.import_skill(SkSearchEngine(), "WebSearchSkill") # role.import_semantic_skill_from_directory(skills_directory, "QASkill") - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) async def action_planner_example(): @@ -75,7 +76,7 @@ async def action_planner_example(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add + await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) # it will choose mathskill.Add if __name__ == "__main__": diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 790295d55..1954e750a 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -16,9 +16,10 @@ from metagpt.logs import logger from metagpt.utils.common import OutputParser from metagpt.utils.custom_decoder import CustomDecoder +from metagpt.utils.named import Named -class Action(ABC): +class Action(ABC, Named): def __init__(self, name: str = "", context=None, llm: LLM = None): self.name: str = name if llm is None: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c000805c5..421211d60 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,13 +5,14 @@ @Author : alexanderwu @File : write_code.py """ +from tenacity import retry, stop_after_attempt, wait_fixed + from metagpt.actions import WriteDesign from metagpt.actions.action import Action from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -55,7 +56,8 @@ def _save(self, context, filename, code): if self._is_invalid(filename): return - design = [i for i in context if i.cause_by == WriteDesign][0] + message_filter = {WriteDesign.get_class_name()} + design = [i for i in context if i.is_recipient(message_filter)][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) ws_path = WORKSPACE_ROOT / ws_name @@ -74,9 +76,8 @@ async def write_code(self, prompt): async def run(self, context, filename): prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f'Writing {filename}..') + logger.info(f"Writing {filename}..") code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) return code - \ No newline at end of file diff --git a/metagpt/const.py b/metagpt/const.py index 7f3f87dfa..3fbc26784 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -41,3 +41,8 @@ def get_project_root(): SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" MEM_TTL = 24 * 30 * 3600 + +MESSAGE_ROUTE_FROM = "tx_from" +MESSAGE_ROUTE_TO = "tx_to" +MESSAGE_ROUTE_CAUSE_BY = "cause_by" +MESSAGE_META_ROLE = "role" diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..ba0645a36 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -4,60 +4,61 @@ @Time : 2023/5/11 22:12 @Author : alexanderwu @File : environment.py +@Modified By: mashenquan, 2023-11-1. Optimization: + 1. Remove the functionality of `Environment` class as a public message buffer. + 2. Standardize the message forwarding behavior of the `Environment` class. + 3. Add the `is_idle` property. """ import asyncio from typing import Iterable from pydantic import BaseModel, Field -from metagpt.memory import Memory +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message class Environment(BaseModel): """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 - Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles + """ roles: dict[str, Role] = Field(default_factory=dict) - memory: Memory = Field(default_factory=Memory) - history: str = Field(default='') class Config: arbitrary_types_allowed = True def add_role(self, role: Role): """增加一个在当前环境的角色 - Add a role in the current environment + Add a role in the current environment """ role.set_env(self) self.roles[role.profile] = role def add_roles(self, roles: Iterable[Role]): """增加一批在当前环境的角色 - Add a batch of characters in the current environment + Add a batch of characters in the current environment """ for role in roles: self.add_role(role) def publish_message(self, message: Message): - """向当前环境发布信息 - Post information to the current environment - """ - # self.message_queue.put(message) - self.memory.add(message) - self.history += f"\n{message}" + """Distribute the message to the recipients.""" + logger.info(f"publish_message: {message.save()}") + found = False + for r in self.roles.values(): + if message.is_recipient(r.subscribed_tags): + r.async_put_message(message) + found = True + if not found: + logger.warning(f"Message no recipients: {message.save()}") async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once """ - # while not self.message_queue.empty(): - # message = self.message_queue.get() - # rsp = await self.manager.handle(message, self) - # self.message_queue.put(rsp) for _ in range(k): futures = [] for role in self.roles.values(): @@ -65,15 +66,24 @@ async def run(self, k=1): futures.append(future) await asyncio.gather(*futures) + logger.info(f"is idle: {self.is_idle}") def get_roles(self) -> dict[str, Role]: """获得环境内的所有角色 - Process all Role runs at once + Process all Role runs at once """ return self.roles def get_role(self, name: str) -> Role: """获得环境内的指定角色 - get all the environment roles + get all the environment roles """ return self.roles.get(name, None) + + @property + def is_idle(self): + """If true, all actions have been executed.""" + for r in self.roles.values(): + if not r.is_idle: + return False + return True diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index f8abea5f3..b5bb73b6b 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -1,6 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the implement of Long-term memory +""" +@Desc : the implement of Long-term memory +@Modified By: mashenquan, 2023-11-1. Optimization: + 1. Replace code related to message filtering with the `Message.is_recipient` function. +""" from metagpt.logs import logger from metagpt.memory import Memory @@ -36,11 +40,10 @@ def recover_memory(self, role_id: str, rc: "RoleContext"): def add(self, message: Message): super(LongTermMemory, self).add(message) - for action in self.rc.watch: - if message.cause_by == action and not self.msg_from_recover: - # currently, only add role's watching messages to its memory_storage - # and ignore adding messages from recover repeatedly - self.memory_storage.add(message) + if message.is_recipient(self.rc.watch) and not self.msg_from_recover: + # currently, only add role's watching messages to its memory_storage + # and ignore adding messages from recover repeatedly + self.memory_storage.add(message) def find_news(self, observed: list[Message], k=0) -> list[Message]: """ @@ -68,4 +71,3 @@ def delete(self, message: Message): def clear(self): super(LongTermMemory, self).clear() self.memory_storage.clean() - \ No newline at end of file diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index c818fa707..8e01544f1 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,11 +4,11 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py +@Modified By: mashenquan, 2023-11-1. Standardize the design of message filtering-related features. """ from collections import defaultdict -from typing import Iterable, Type +from typing import Iterable, Set -from metagpt.actions import Action from metagpt.schema import Message @@ -18,7 +18,7 @@ class Memory: def __init__(self): """Initialize an empty storage list and an empty index dictionary""" self.storage: list[Message] = [] - self.index: dict[Type[Action], list[Message]] = defaultdict(list) + self.index: dict[str, list[Message]] = defaultdict(list) def add(self, message: Message): """Add a new message to storage, while updating the index""" @@ -73,11 +73,11 @@ def find_news(self, observed: list[Message], k=0) -> list[Message]: news.append(i) return news - def get_by_action(self, action: Type[Action]) -> list[Message]: + def get_by_action(self, action: str) -> list[Message]: """Return all messages triggered by a specified Action""" return self.index[action] - def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: + def get_by_actions(self, actions: Set[str]) -> list[Message]: """Return all messages triggered by specified Actions""" rsp = [] for action in actions: @@ -85,4 +85,3 @@ def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: continue rsp += self.index[action] return rsp - \ No newline at end of file diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 6d65575a8..9826ea0b7 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -4,6 +4,10 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : engineer.py +@Modified By: mashenquan, 2023-11-1. Optimization: + 1. Consolidate message reception and processing logic within `_observe`. + 2. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. + 3. Supplemented the external transmission of internal messages. """ import asyncio import shutil @@ -15,7 +19,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import CodeParser +from metagpt.utils.common import CodeParser, get_object_name from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -75,7 +79,7 @@ def __init__( self.use_code_review = use_code_review if self.use_code_review: self._init_actions([WriteCode, WriteCodeReview]) - self._watch([WriteTasks]) + self._watch([WriteTasks, WriteDesign]) self.todos = [] self.n_borg = n_borg @@ -96,7 +100,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] + msg = self._rc.memory.get_by_action(WriteDesign.get_class_name())[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -119,17 +123,13 @@ def write_file(self, filename: str, code: str): file.write_text(code) return file - def recv(self, message: Message) -> None: - self._rc.memory.add(message) - if message in self._rc.important_memory: - self.todos = self.parse_tasks(message) - async def _act_mp(self) -> Message: # self.recreate_workspace() todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo + context=self._rc.memory.get_by_actions([WriteTasks.get_class_name(), WriteDesign.get_class_name()]), + filename=todo, ) todo_coros.append(todo_coro) @@ -139,12 +139,13 @@ async def _act_mp(self) -> Message: logger.info(todo) logger.info(code_rsp) # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content=code_rsp, role=self.profile, cause_by=get_object_name(self._rc.todo)) self._rc.memory.add(msg) + self.publish_message(msg) del self.todos[0] logger.info(f"Done {self.get_workspace()} generating.") - msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content="all done.", role=self.profile, cause_by=get_object_name(self._rc.todo)) return msg async def _act_sp(self) -> Message: @@ -155,15 +156,19 @@ async def _act_sp(self) -> Message: # logger.info(code_rsp) # code = self.parse_code(code_rsp) file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content=code, role=self.profile, cause_by=get_object_name(self._rc.todo)) self._rc.memory.add(msg) + self.publish_message(msg) code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" + content=MSG_SEP.join(code_msg_all), + role=self.profile, + cause_by=get_object_name(self._rc.todo), + tx_to="QaEngineer", ) return msg @@ -178,7 +183,8 @@ async def _act_sp_precision(self) -> Message: TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + msg_filters = [WriteDesign.get_class_name(), WriteTasks.get_class_name(), WriteCode.get_class_name()] + msg = self._rc.memory.get_by_actions(msg_filters) for m in msg: context.append(m.content) context_str = "\n".join(context) @@ -193,20 +199,50 @@ async def _act_sp_precision(self) -> Message: logger.error("code review failed!", e) pass file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) + msg = Message(content=code, role=self.profile, cause_by=WriteCode.get_class_name()) self._rc.memory.add(msg) + self.publish_message(msg) code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" + content=MSG_SEP.join(code_msg_all), + role=self.profile, + cause_by=get_object_name(self._rc.todo), + tx_to="QaEngineer", ) return msg async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" + if not self._rc.todo: + return None if self.use_code_review: return await self._act_sp_precision() return await self._act_sp() + + async def _observe(self) -> int: + ret = await super(Engineer, self)._observe() + if ret == 0: + return ret + + # Parse task lists + message_filter = {WriteTasks.get_class_name()} + for message in self._rc.news: + if not message.is_recipient(message_filter): + continue + self.todos = self.parse_tasks(message) + + return ret + + async def _think(self) -> None: + # In asynchronous scenarios, first check if the required messages are ready. + filters = {WriteTasks.get_class_name()} + msgs = self._rc.memory.get_by_actions(filters) + if not msgs: + self._rc.todo = None + return + + await super(Engineer, self)._think() diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a763c2ce8..b83ab6e21 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : qa_engineer.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. """ import os from pathlib import Path @@ -48,7 +49,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] + msg = self._rc.memory.get_by_action(WriteDesign.get_class_name())[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -97,11 +98,11 @@ async def _write_test(self, message: Message) -> None: msg = Message( content=str(file_info), role=self.profile, - cause_by=WriteTest, - sent_from=self.profile, - send_to=self.profile, + cause_by=WriteTest.get_class_name(), + tx_from=self.profile, + tx_to=self.profile, ) - self._publish_message(msg) + self.publish_message(msg) logger.info(f"Done {self.get_workspace()}/tests generating.") @@ -131,8 +132,10 @@ async def _run_code(self, msg): recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message(content=content, role=self.profile, cause_by=RunCode, sent_from=self.profile, send_to=recipient) - self._publish_message(msg) + msg = Message( + content=content, role=self.profile, cause_by=RunCode.get_class_name(), tx_from=self.profile, tx_to=recipient + ) + self.publish_message(msg) async def _debug_error(self, msg): file_info, context = msg.content.split(FILENAME_CODE_SEP) @@ -141,14 +144,18 @@ async def _debug_error(self, msg): self.write_file(file_name, code) recipient = msg.sent_from # send back to the one who ran the code for another run, might be one's self msg = Message( - content=file_info, role=self.profile, cause_by=DebugError, sent_from=self.profile, send_to=recipient + content=file_info, + role=self.profile, + cause_by=DebugError.get_class_name(), + tx_from=self.profile, + tx_to=recipient, ) - self._publish_message(msg) + self.publish_message(msg) async def _observe(self) -> int: await super()._observe() self._rc.news = [ - msg for msg in self._rc.news if msg.send_to == self.profile + msg for msg in self._rc.news if msg.is_recipient({self.profile}) ] # only relevant msgs count as observed news return len(self._rc.news) @@ -157,30 +164,31 @@ async def _act(self) -> Message: result_msg = Message( content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", role=self.profile, - cause_by=WriteTest, - sent_from=self.profile, - send_to="", + cause_by=WriteTest.get_class_name(), + tx_from=self.profile, ) return result_msg + code_filters = {WriteCode.get_class_name(), WriteCodeReview.get_class_name()} + test_filters = {WriteTest.get_class_name(), DebugError.get_class_name()} + run_filters = {RunCode.get_class_name()} for msg in self._rc.news: # Decide what to do based on observed msg type, currently defined by human, # might potentially be moved to _think, that is, let the agent decides for itself - if msg.cause_by in [WriteCode, WriteCodeReview]: + if msg.is_recipient(code_filters): # engineer wrote a code, time to write a test for it await self._write_test(msg) - elif msg.cause_by in [WriteTest, DebugError]: + elif msg.is_recipient(test_filters): # I wrote or debugged my test code, time to run it await self._run_code(msg) - elif msg.cause_by == RunCode: + elif msg.is_recipient(run_filters): # I ran my test code, time to fix bugs, if any await self._debug_error(msg) self.test_round += 1 result_msg = Message( content=f"Round {self.test_round} of tests done", role=self.profile, - cause_by=WriteTest, - sent_from=self.profile, - send_to="", + cause_by=WriteTest.get_class_name(), + tx_from=self.profile, ) return result_msg diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index acb46c718..43ee7971d 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -1,4 +1,8 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +""" + import asyncio @@ -10,6 +14,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message +from metagpt.utils.common import get_object_name class Report(BaseModel): @@ -58,18 +63,22 @@ async def _act(self) -> Message: research_system_text = get_research_system_text(topic, self.language) if isinstance(todo, CollectLinks): links = await todo.run(topic, 4, 4) - ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=type(todo)) + ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=get_object_name(todo)) elif isinstance(todo, WebBrowseAndSummarize): links = instruct_content.links todos = (todo.run(*url, query=query, system_text=research_system_text) for (query, url) in links.items()) summaries = await asyncio.gather(*todos) summaries = list((url, summary) for i in summaries for (url, summary) in i.items() if summary) - ret = Message("", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=type(todo)) + ret = Message( + "", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=get_object_name(todo) + ) else: summaries = instruct_content.summaries summary_text = "\n---\n".join(f"url: {url}\nsummary: {summary}" for (url, summary) in summaries) content = await self._rc.todo.run(topic, summary_text, system_text=research_system_text) - ret = Message("", Report(topic=topic, content=content), role=self.profile, cause_by=type(self._rc.todo)) + ret = Message( + "", Report(topic=topic, content=content), role=self.profile, get_object_name=type(self._rc.todo) + ) self._rc.memory.add(ret) return ret diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 44bb3e976..0a6716428 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,20 +4,32 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py +@Modified By: mashenquan, 2023-11-1. Optimization: + 1. Merge the `recv` functionality into the `_observe` function. Future message reading operations will be + consolidated within the `_observe` function. + 2. Standardize the message filtering for string label matching. Role objects can access the message labels + they've subscribed to through the `subscribed_tags` property. + 3. Move the message receive buffer from the global variable `self._rc.env.memory` to the role's private variable + `self._rc.msg_buffer` for easier message identification and asynchronous appending of messages. + 4. Standardize the way messages are passed: `publish_message` sends messages out, while `async_put_message` places + messages into the Role object's private message receive buffer. There are no other message transmit methods. + 5. Standardize the parameters for the `run` function: the `test_message` parameter is used for testing purposes + only. In the normal workflow, you should use `publish_message` or `async_put_message` to transmit messages. """ from __future__ import annotations -from typing import Iterable, Type +from typing import Iterable, Set, Type from pydantic import BaseModel, Field -# from metagpt.environment import Environment -from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG from metagpt.llm import LLM from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory -from metagpt.schema import Message +from metagpt.memory import LongTermMemory, Memory +from metagpt.schema import Message, MessageQueue +from metagpt.utils.common import get_class_name, get_object_name +from metagpt.utils.named import Named PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -49,6 +61,7 @@ class RoleSetting(BaseModel): """Role Settings""" + name: str profile: str goal: str @@ -64,12 +77,14 @@ def __repr__(self): class RoleContext(BaseModel): """Role Runtime Context""" - env: 'Environment' = Field(default=None) + + env: "Environment" = Field(default=None) + msg_buffer: MessageQueue = Field(default_factory=MessageQueue) # Message Buffer with Asynchronous Updates memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=0) todo: Action = Field(default=None) - watch: set[Type[Action]] = Field(default_factory=set) + watch: set[str] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) class Config: @@ -90,7 +105,7 @@ def history(self) -> list[Message]: return self.memory.get() -class Role: +class Role(Named): """Role/Agent""" def __init__(self, name="", profile="", goal="", constraints="", desc=""): @@ -118,7 +133,8 @@ def _init_actions(self, actions): def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" - self._rc.watch.update(actions) + tags = [get_class_name(t) for t in actions] + self._rc.watch.update(tags) # check RoleContext after adding watch actions self._rc.check(self._role_id) @@ -128,7 +144,7 @@ def _set_state(self, state): logger.debug(self._actions) self._rc.todo = self._actions[self._rc.state] - def set_env(self, env: 'Environment'): + def set_env(self, env: "Environment"): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env @@ -137,6 +153,24 @@ def profile(self): """Get the role description (position)""" return self._setting.profile + @property + def name(self): + """Get virtual user name""" + return self._setting.name + + @property + def subscribed_tags(self) -> Set: + """The labels for messages to be consumed by the Role object.""" + if self._rc.watch: + return self._rc.watch + return { + self.name, + self.get_object_name(), + self.profile, + f"{self.name}({self.profile})", + f"{self.name}({self.get_object_name()})", + } + def _get_prefix(self): """Get the role prefix""" if self._setting.desc: @@ -150,94 +184,99 @@ async def _think(self) -> None: self._set_state(0) return prompt = self._get_prefix() - prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + prompt += STATE_TEMPLATE.format( + history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1 + ) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') + logger.warning(f"Invalid answer of state, {next_state=}") next_state = "0" self._set_state(int(next_state)) async def _act(self) -> Message: - # prompt = self.get_prefix() - # prompt += ROLE_TEMPLATE.format(name=self.profile, state=self.states[self.state], result=response, - # history=self.history) - logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.important_memory) - # logger.info(response) if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=get_object_name(self._rc.todo), + tx_from=get_object_name(self), + ) else: - msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) - self._rc.memory.add(msg) - # logger.debug(f"{response}") + msg = Message( + content=response, + role=self.profile, + cause_by=get_object_name(self._rc.todo), + tx_from=get_object_name(self), + ) return msg async def _observe(self) -> int: - """Observe from the environment, obtain important information, and add it to memory""" - if not self._rc.env: - return 0 - env_msgs = self._rc.env.memory.get() - - observed = self._rc.env.memory.get_by_actions(self._rc.watch) - - self._rc.news = self._rc.memory.find_news(observed) # find news (previously unseen messages) from observed messages - - for i in env_msgs: - self.recv(i) - + """Prepare new messages for processing from the message buffer and other sources.""" + # Read unprocessed messages from the msg buffer. + self._rc.news = self._rc.msg_buffer.pop_all() + # Store the read messages in your own memory to prevent duplicate processing. + self._rc.memory.add_batch(self._rc.news) + + # Design Rules: + # If you need to further categorize Message objects, you can do so using the Message.set_meta function. + # msg_buffer is a receiving buffer, avoid adding message data and operations to msg_buffer. news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: - logger.debug(f'{self._setting} observed: {news_text}') + logger.debug(f"{self._setting} observed: {news_text}") return len(self._rc.news) - def _publish_message(self, msg): + def publish_message(self, msg): """If the role belongs to env, then the role's messages will be broadcast to env""" + if not msg: + return if not self._rc.env: # If env does not exist, do not publish the message return self._rc.env.publish_message(msg) + def async_put_message(self, message): + """Place the message into the Role object's private message buffer.""" + if not message: + return + self._rc.msg_buffer.push(message) + async def _react(self) -> Message: """Think first, then act""" await self._think() logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") return await self._act() - def recv(self, message: Message) -> None: - """add message to history.""" - # self._history += f"\n{message}" - # self._context = self._history - if message in self._rc.memory.get(): - return - self._rc.memory.add(message) - - async def handle(self, message: Message) -> Message: - """Receive information and reply with actions""" - # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") - self.recv(message) - - return await self._react() - - async def run(self, message=None): + async def run(self, test_message=None): """Observe, and think and act based on the results of the observation""" - if message: - if isinstance(message, str): - message = Message(message) - if isinstance(message, Message): - self.recv(message) - if isinstance(message, list): - self.recv(Message("\n".join(message))) - elif not await self._observe(): + if test_message: # For test + seed = None + if isinstance(test_message, str): + seed = Message(test_message) + elif isinstance(test_message, Message): + seed = test_message + elif isinstance(test_message, list): + seed = Message("\n".join(test_message)) + self.async_put_message(seed) + + if not await self._observe(): # If there is no new information, suspend and wait logger.debug(f"{self._setting}: no news. waiting.") return rsp = await self._react() - # Publish the reply to the environment, waiting for the next subscriber to process - self._publish_message(rsp) + + # Reset the next action to be taken. + self._rc.todo = None + # Send the response message to the Environment object to have it relay the message to the subscribers. + self.publish_message(rsp) return rsp + + @property + def is_idle(self) -> bool: + """If true, all actions have been executed.""" + return not self._rc.news and not self._rc.todo and self._rc.msg_buffer.empty() diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index 0b6e089da..95be89277 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -4,18 +4,20 @@ @Time : 2023/5/23 17:25 @Author : alexanderwu @File : seacher.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. """ from metagpt.actions import ActionOutput, SearchAndSummarize from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message from metagpt.tools import SearchEngineType +from metagpt.utils.common import get_object_name class Searcher(Role): """ Represents a Searcher role responsible for providing search services to users. - + Attributes: name (str): Name of the searcher. profile (str): Role profile. @@ -23,17 +25,19 @@ class Searcher(Role): constraints (str): Constraints or limitations for the searcher. engine (SearchEngineType): The type of search engine to use. """ - - def __init__(self, - name: str = 'Alice', - profile: str = 'Smart Assistant', - goal: str = 'Provide search services for users', - constraints: str = 'Answer is rich and complete', - engine=SearchEngineType.SERPAPI_GOOGLE, - **kwargs) -> None: + + def __init__( + self, + name: str = "Alice", + profile: str = "Smart Assistant", + goal: str = "Provide search services for users", + constraints: str = "Answer is rich and complete", + engine=SearchEngineType.SERPAPI_GOOGLE, + **kwargs, + ) -> None: """ Initializes the Searcher role with given attributes. - + Args: name (str): Name of the searcher. profile (str): Role profile. @@ -53,12 +57,16 @@ async def _act_sp(self) -> Message: """Performs the search action in a single process.""" logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.memory.get(k=0)) - + if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=get_object_name(self._rc.todo), + ) else: - msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content=response, role=self.profile, cause_by=get_object_name(self._rc.todo)) self._rc.memory.add(msg) return msg diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index b27841d74..abebb9605 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -4,6 +4,7 @@ @Time : 2023/9/13 12:23 @Author : femto Zheng @File : sk_agent.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. """ from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner @@ -14,6 +15,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message +from metagpt.utils.common import get_object_name from metagpt.utils.make_sk_kernel import make_sk_kernel @@ -70,7 +72,7 @@ async def _act(self) -> Message: result = (await self.plan.invoke_async()).result logger.info(result) - msg = Message(content=result, role=self.profile, cause_by=type(self._rc.todo)) + msg = Message(content=result, role=self.profile, cause_by=get_object_name(self._rc.todo)) self._rc.memory.add(msg) - # logger.debug(f"{response}") + self.publish_message(msg) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 1124fb28e..e0d17e0ed 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -8,12 +8,20 @@ """ from __future__ import annotations +import asyncio import json +from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError -from typing import Dict, List, TypedDict +from typing import Dict, List, Set, TypedDict from pydantic import BaseModel, Field +from metagpt.const import ( + MESSAGE_META_ROLE, + MESSAGE_ROUTE_CAUSE_BY, + MESSAGE_ROUTE_FROM, + MESSAGE_ROUTE_TO, +) from metagpt.logs import logger @@ -22,44 +30,150 @@ class RawMessage(TypedDict): role: str +class Routes(BaseModel): + """Responsible for managing routing information for the Message class.""" + + routes: List[Dict] = Field(default_factory=list) + + def set_from(self, value): + """Set the label of the message sender.""" + route = self._get_route() + route[MESSAGE_ROUTE_FROM] = value + + def set_to(self, tags: Set): + """Set the labels of the message recipient.""" + route = self._get_route() + if tags: + route[MESSAGE_ROUTE_TO] = tags + return + + if MESSAGE_ROUTE_TO in route: + del route[MESSAGE_ROUTE_TO] + + def add_to(self, tag: str): + """Add a label of the message recipient.""" + route = self._get_route() + tags = route.get(MESSAGE_ROUTE_TO, set()) + tags.add(tag) + route[MESSAGE_ROUTE_TO] = tags + + def _get_route(self) -> Dict: + if not self.routes: + self.routes.append({}) + return self.routes[0] + + def is_recipient(self, tags: Set) -> bool: + """Check if it is the message recipient.""" + route = self._get_route() + to_tags = route.get(MESSAGE_ROUTE_TO) + if not to_tags: + return True + + for k in tags: + if k in to_tags: + return True + return False + + @property + def tx_from(self): + """Message route info tells who sent this message.""" + route = self._get_route() + return route.get(MESSAGE_ROUTE_FROM) + + @property + def tx_to(self): + """Labels for the consumer to filter its subscribed messages.""" + route = self._get_route() + return route.get(MESSAGE_ROUTE_TO) + + class Message(BaseModel): """list[: ]""" content: str instruct_content: BaseModel = None meta_info: Dict = Field(default_factory=dict) - route: List[Dict] = Field(default_factory=list) + route: Routes = Field(default_factory=Routes) def __init__(self, content, **kwargs): + """ + :param content: Message content. + :param instruct_content: Message content struct. + :param meta_info: Message meta info. + :param route: Message route configuration. + :param tx_from: Message route info tells who sent this message. + :param tx_to: Labels for the consumer to filter its subscribed messages. + :param cause_by: Labels for the consumer to filter its subscribed messages, also serving as meta info. + :param role: Message meta info tells who sent this message. + """ super(Message, self).__init__( content=content or kwargs.get("content"), instruct_content=kwargs.get("instruct_content"), meta_info=kwargs.get("meta_info", {}), - route=kwargs.get("route", []), + route=kwargs.get("route", Routes()), ) attribute_names = Message.__annotations__.keys() for k, v in kwargs.items(): if k in attribute_names: continue + if k == MESSAGE_ROUTE_FROM: + self.set_from(v) + continue + if k == MESSAGE_ROUTE_CAUSE_BY: + self.meta_info[k] = v + if k == MESSAGE_ROUTE_TO or k == MESSAGE_ROUTE_CAUSE_BY: + self.add_to(v) + continue self.meta_info[k] = v def get_meta(self, key): + """Get meta info""" return self.meta_info.get(key) def set_meta(self, key, value): + """Set meta info""" self.meta_info[key] = value @property def role(self): - return self.get_meta("role") + """Message meta info tells who sent this message.""" + return self.get_meta(MESSAGE_META_ROLE) @property def cause_by(self): - return self.get_meta("cause_by") + """Labels for the consumer to filter its subscribed messages, also serving as meta info.""" + return self.get_meta(MESSAGE_ROUTE_CAUSE_BY) + + @property + def tx_from(self): + """Message route info tells who sent this message.""" + return self.route.tx_from + + @property + def tx_to(self): + """Labels for the consumer to filter its subscribed messages.""" + return self.route.tx_to def set_role(self, v): - self.set_meta("role", v) + """Set the message's meta info indicating the sender.""" + self.set_meta(MESSAGE_META_ROLE, v) + + def set_from(self, v): + """Set the message's meta info indicating the sender.""" + self.route.set_from(v) + + def set_to(self, tags: Set): + """Set the message's meta info indicating the sender.""" + self.route.set_to(tags) + + def add_to(self, tag: str): + """Add a subscription label for the recipients.""" + self.route.add_to(tag) + + def is_recipient(self, tags: Set): + """Return true if any input label exists in the message's subscription labels.""" + return self.route.is_recipient(tags) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) @@ -69,13 +183,16 @@ def __repr__(self): return self.__str__() def to_dict(self) -> dict: + """Return a dict containing `role` and `content` for the LLM call.l""" return {"role": self.role, "content": self.content} def save(self) -> str: + """Convert the object to json string""" return self.json(exclude_none=True) @staticmethod def load(v): + """Convert the json string to object.""" try: d = json.loads(v) return Message(**d) @@ -90,7 +207,7 @@ class UserMessage(Message): """ def __init__(self, content: str): - super(Message, self).__init__(content=content, meta_info={"role": "user"}) + super().__init__(content=content, role="user") class SystemMessage(Message): @@ -99,7 +216,7 @@ class SystemMessage(Message): """ def __init__(self, content: str): - super().__init__(content=content, meta_info={"role": "system"}) + super().__init__(content=content, role="system") class AIMessage(Message): @@ -108,7 +225,65 @@ class AIMessage(Message): """ def __init__(self, content: str): - super().__init__(content=content, meta_info={"role": "assistant"}) + super().__init__(content=content, role="assistant") + + +class MessageQueue: + def __init__(self): + self._queue = Queue() + + def pop(self) -> Message | None: + try: + item = self._queue.get_nowait() + if item: + self._queue.task_done() + return item + except QueueEmpty: + return None + + def pop_all(self) -> List[Message]: + ret = [] + while True: + msg = self.pop() + if not msg: + break + ret.append(msg) + return ret + + def push(self, msg: Message): + self._queue.put_nowait(msg) + + def empty(self): + return self._queue.empty() + + async def save(self) -> str: + if self.empty(): + return "[]" + + lst = [] + try: + while True: + item = await wait_for(self._queue.get(), timeout=1.0) + if item is None: + break + lst.append(item.dict(exclude_none=True)) + self._queue.task_done() + except asyncio.TimeoutError: + logger.debug("Queue is empty, exiting...") + return json.dumps(lst) + + @staticmethod + def load(self, v) -> "MessageQueue": + q = MessageQueue() + try: + lst = json.loads(v) + for i in lst: + msg = Message(**i) + q.push(msg) + except JSONDecodeError as e: + logger.warning(f"JSON load failed: {v}, error:{e}") + + return q if __name__ == "__main__": diff --git a/metagpt/software_company.py b/metagpt/software_company.py index b2bd18c58..4bedec0e1 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,6 +4,9 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023-11-1. Optimization: + 1. Standardize the design of message filtering-related features. + 2. Abandon the design of having `Environment` store all messages. """ from pydantic import BaseModel, Field @@ -14,13 +17,15 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException +from metagpt.utils.named import Named -class SoftwareCompany(BaseModel): +class SoftwareCompany(BaseModel, Named): """ Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to writing executable code. """ + environment: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -36,16 +41,23 @@ def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment CONFIG.max_budget = investment - logger.info(f'Investment: ${investment}.') + logger.info(f"Investment: ${investment}.") def _check_balance(self): if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') + raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}") def start_project(self, idea): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) + self.environment.publish_message( + Message( + role="BOSS", + content=idea, + cause_by=BossRequirement.get_class_name(), + tx_from=SoftwareCompany.get_class_name(), + ) + ) def _save(self): logger.info(self.json()) @@ -58,5 +70,3 @@ async def run(self, n_round=3): logger.debug(f"{n_round=}") self._check_balance() await self.environment.run() - return self.environment.history - \ No newline at end of file diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index f09666beb..df4688378 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -85,10 +85,7 @@ def parse_file_list(cls, text: str) -> list[str]: @staticmethod def parse_python_code(text: str) -> str: - for pattern in ( - r"(.*?```python.*?\s+)?(?P.*)(```.*?)", - r"(.*?```python.*?\s+)?(?P.*)", - ): + for pattern in (r"(.*?```python.*?\s+)?(?P.*)(```.*?)", r"(.*?```python.*?\s+)?(?P.*)"): match = re.search(pattern, text, re.DOTALL) if not match: continue @@ -305,3 +302,14 @@ def parse_recipient(text): pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now recipient = re.search(pattern, text) return recipient.group(1) if recipient else "" + + +def get_class_name(cls) -> str: + """Return class name""" + return f"{cls.__module__}.{cls.__name__}" + + +def get_object_name(obj) -> str: + """Return class name of the object""" + cls = type(obj) + return f"{cls.__module__}.{cls.__name__}" diff --git a/metagpt/utils/named.py b/metagpt/utils/named.py new file mode 100644 index 000000000..e4da574e8 --- /dev/null +++ b/metagpt/utils/named.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/1 +@Author : mashenquan +@File : named.py +""" + + +class Named: + """A base class with functions for converting classes to names and objects to class names.""" + + @classmethod + def get_class_name(cls): + """Return class name""" + return f"{cls.__module__}.{cls.__name__}" + + def get_object_name(self): + """Return class name of the object""" + cls = type(self) + return f"{cls.__module__}.{cls.__name__}" diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 38e4e5221..40ab20dad 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_prd.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. """ import pytest @@ -17,7 +18,7 @@ async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement)) + prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement.get_class_name())) logger.info(requirements) logger.info(prd) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index dc5540520..c40d7ab9d 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -1,12 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : unittest of `metagpt/memory/longterm_memory.py` +""" +@Desc : unittest of `metagpt/memory/longterm_memory.py` +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +""" -from metagpt.config import CONFIG -from metagpt.schema import Message from metagpt.actions import BossRequirement -from metagpt.roles.role import RoleContext +from metagpt.config import CONFIG from metagpt.memory import LongTermMemory +from metagpt.roles.role import RoleContext +from metagpt.schema import Message def test_ltm_search(): @@ -14,25 +17,25 @@ def test_ltm_search(): openai_api_key = CONFIG.openai_api_key assert len(openai_api_key) > 20 - role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + role_id = "UTUserLtm(Product Manager)" + rc = RoleContext(watch=[BossRequirement.get_class_name()]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) - idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + message = Message(role="BOSS", content=idea, cause_by=BossRequirement.get_class_name()) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement.get_class_name()) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement.get_class_name()) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -47,8 +50,8 @@ def test_ltm_search(): news = ltm_new.find_news([sim_message]) assert len(news) == 0 - new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a Battle City" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement.get_class_name()) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 6bb3e8f1d..881b47d6f 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -1,20 +1,23 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the unittests of metagpt/memory/memory_storage.py +""" +@Desc : the unittests of metagpt/memory/memory_storage.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +""" + from typing import List +from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions.action_output import ActionOutput from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message -from metagpt.actions import BossRequirement -from metagpt.actions import WritePRD -from metagpt.actions.action_output import ActionOutput def test_idea_message(): - idea = 'Write a cli snake game' - role_id = 'UTUser1(Product Manager)' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + role_id = "UTUser1(Product Manager)" + message = Message(role="BOSS", content=idea, cause_by=BossRequirement.get_class_name()) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -23,13 +26,13 @@ def test_idea_message(): memory_storage.add(message) assert memory_storage.is_initialized is True - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement.get_class_name()) new_messages = memory_storage.search(sim_message) - assert len(new_messages) == 0 # similar, return [] + assert len(new_messages) == 0 # similar, return [] - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement.get_class_name()) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -38,22 +41,15 @@ def test_idea_message(): def test_actionout_message(): - out_mapping = { - 'field1': (str, ...), - 'field2': (List[str], ...) - } - out_data = { - 'field1': 'field1 value', - 'field2': ['field2 value1', 'field2 value2'] - } - ic_obj = ActionOutput.create_model_class('prd', out_mapping) - - role_id = 'UTUser2(Architect)' - content = 'The boss has requested the creation of a command-line interface (CLI) snake game' - message = Message(content=content, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) # WritePRD as test action + out_mapping = {"field1": (str, ...), "field2": (List[str], ...)} + out_data = {"field1": "field1 value", "field2": ["field2 value1", "field2 value2"]} + ic_obj = ActionOutput.create_model_class("prd", out_mapping) + + role_id = "UTUser2(Architect)" + content = "The boss has requested the creation of a command-line interface (CLI) snake game" + message = Message( + content=content, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() + ) # WritePRD as test action memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -62,19 +58,17 @@ def test_actionout_message(): memory_storage.add(message) assert memory_storage.is_initialized is True - sim_conent = 'The request is command-line interface (CLI) snake game' - sim_message = Message(content=sim_conent, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) + sim_conent = "The request is command-line interface (CLI) snake game" + sim_message = Message( + content=sim_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() + ) new_messages = memory_storage.search(sim_message) - assert len(new_messages) == 0 # similar, return [] + assert len(new_messages) == 0 # similar, return [] - new_conent = 'Incorporate basic features of a snake game such as scoring and increasing difficulty' - new_message = Message(content=new_conent, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) + new_conent = "Incorporate basic features of a snake game such as scoring and increasing difficulty" + new_message = Message( + content=new_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() + ) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index 5ab9a493f..a3831c08d 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -4,6 +4,9 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py +@Modified By: mashenquan, 2023-11-1. Optimization: + 1. Standardize the usage of message filtering-related features. + 2. Standardize the usage of message transmission. """ import pytest from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill @@ -23,7 +26,7 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.recv(Message(content=task, cause_by=BossRequirement)) - + role.async_put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) + await role._observe() await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 03a82ec5e..9efcb9367 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -4,6 +4,9 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py +@Modified By: mashenquan, 2023-11-1. Optimization: + 1. Standardize the usage of message filtering-related features. + 2. Standardize the usage of message transmission. """ import pytest from semantic_kernel.core_skills import TextSkill @@ -26,7 +29,8 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.recv(Message(content=task, cause_by=BossRequirement)) + role.async_put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) + await role._observe() await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate assert "WriterSkill.Brainstorm" in role.plan.generated_plan.result diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 52fc4a3c1..b9891cd81 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -4,6 +4,7 @@ @Time : 2023/5/12 13:05 @Author : alexanderwu @File : mock.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. """ from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message @@ -71,7 +72,7 @@ ``` ''' -SYSTEM_DESIGN = '''## Python package name +SYSTEM_DESIGN = """## Python package name ```python "smart_search_engine" ``` @@ -149,10 +150,10 @@ class KnowledgeBase { S-->>SE: return summary SE-->>M: return summary ``` -''' +""" -TASKS = '''## Logic Analysis +TASKS = """## Logic Analysis 在这个项目中，所有的模块都依赖于“SearchEngine”类，这是主入口，其他的模块（Index、Ranking和Summary）都通过它交互。另外，"Index"类又依赖于"KnowledgeBase"类，因为它需要从知识库中获取数据。 @@ -181,7 +182,7 @@ class KnowledgeBase { ] ``` 这个任务列表首先定义了最基础的模块，然后是依赖这些模块的模块，最后是辅助模块。可以根据团队的能力和资源，同时开发多个任务，只要满足依赖关系。例如，在开发"search.py"之前，可以同时开发"knowledge_base.py"、"index.py"、"ranking.py"和"summary.py"。 -''' +""" TASKS_TOMATO_CLOCK = '''## Required Python third-party packages: Provided in requirements.txt format @@ -224,35 +225,35 @@ class KnowledgeBase { TASK = """smart_search_engine/knowledge_base.py""" STRS_FOR_PARSING = [ -""" + """ ## 1 ```python a ``` """, -""" + """ ##2 ```python "a" ``` """, -""" + """ ## 3 ```python a = "a" ``` """, -""" + """ ## 4 ```python a = 'a' ``` -""" +""", ] class MockMessages: - req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement) - prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD) - system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign) - tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks) + req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement.get_class_name()) + prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD.get_class_name()) + system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign.get_class_name()) + tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks.get_class_name()) diff --git a/tests/metagpt/roles/test_architect.py b/tests/metagpt/roles/test_architect.py index d44e0d923..910c589ca 100644 --- a/tests/metagpt/roles/test_architect.py +++ b/tests/metagpt/roles/test_architect.py @@ -4,6 +4,7 @@ @Time : 2023/5/20 14:37 @Author : alexanderwu @File : test_architect.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message transmission. """ import pytest @@ -15,7 +16,7 @@ @pytest.mark.asyncio async def test_architect(): role = Architect() - role.recv(MockMessages.req) - rsp = await role.handle(MockMessages.prd) + role.async_put_message(MockMessages.req) + rsp = await role.run(MockMessages.prd) logger.info(rsp) assert len(rsp.content) > 0 diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index c0c48d0b1..e80234b3b 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -4,6 +4,7 @@ @Time : 2023/5/12 10:14 @Author : alexanderwu @File : test_engineer.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message transmission. """ import pytest @@ -22,10 +23,10 @@ async def test_engineer(): engineer = Engineer() - engineer.recv(MockMessages.req) - engineer.recv(MockMessages.prd) - engineer.recv(MockMessages.system_design) - rsp = await engineer.handle(MockMessages.tasks) + engineer.async_put_message(MockMessages.req) + engineer.async_put_message(MockMessages.prd) + engineer.async_put_message(MockMessages.system_design) + rsp = await engineer.run(MockMessages.tasks) logger.info(rsp) assert "all done." == rsp.content @@ -35,13 +36,13 @@ def test_parse_str(): for idx, i in enumerate(STRS_FOR_PARSING): text = CodeParser.parse_str(f"{idx+1}", i) # logger.info(text) - assert text == 'a' + assert text == "a" def test_parse_blocks(): tasks = CodeParser.parse_blocks(TASKS) logger.info(tasks.keys()) - assert 'Task list' in tasks.keys() + assert "Task list" in tasks.keys() target_list = [ diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..755798b17 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -4,6 +4,7 @@ @Time : 2023/5/12 00:47 @Author : alexanderwu @File : test_environment.py +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message transmission. """ import pytest @@ -49,7 +50,7 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) env.set_manager(Manager()) - env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) + env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement.get_class_name())) await env.run(k=2) logger.info(f"{env.history=}") diff --git a/tests/metagpt/utils/test_serialize.py b/tests/metagpt/utils/test_serialize.py index 69f317f79..5a0840c87 100644 --- a/tests/metagpt/utils/test_serialize.py +++ b/tests/metagpt/utils/test_serialize.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the unittest of serialize +""" +@Desc : the unittest of serialize +@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +""" from typing import List, Tuple @@ -55,7 +58,7 @@ def test_serialize_and_deserialize_message(): ic_obj = ActionOutput.create_model_class("prd", out_mapping) message = Message( - content="prd demand", instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD + content="prd demand", instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() ) # WritePRD as test action message_ser = serialize_message(message) From bd813d2b90d16d2c439c4693ab27dca595786b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 1 Nov 2023 20:17:23 +0800 Subject: [PATCH 004/232] refactor: Refactor Message transmission & filtering --- metagpt/roles/researcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 43ee7971d..6e89b9fe7 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -77,7 +77,7 @@ async def _act(self) -> Message: summary_text = "\n---\n".join(f"url: {url}\nsummary: {summary}" for (url, summary) in summaries) content = await self._rc.todo.run(topic, summary_text, system_text=research_system_text) ret = Message( - "", Report(topic=topic, content=content), role=self.profile, get_object_name=type(self._rc.todo) + "", Report(topic=topic, content=content), role=self.profile, cause_by=get_object_name(self._rc.todo) ) self._rc.memory.add(ret) return ret From 8582f219624d75440d33e32648ebf1b44c389011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 1 Nov 2023 20:33:34 +0800 Subject: [PATCH 005/232] refactor: Refactor Message transmission & filtering --- metagpt/schema.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index e0d17e0ed..806b0e94e 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -229,10 +229,13 @@ def __init__(self, content: str): class MessageQueue: + """Message queue which supports asynchronous updates.""" + def __init__(self): self._queue = Queue() def pop(self) -> Message | None: + """Pop one message from queue.""" try: item = self._queue.get_nowait() if item: @@ -242,6 +245,7 @@ def pop(self) -> Message | None: return None def pop_all(self) -> List[Message]: + """Pop all messages from queue.""" ret = [] while True: msg = self.pop() @@ -251,12 +255,15 @@ def pop_all(self) -> List[Message]: return ret def push(self, msg: Message): + """Push a message into the queue.""" self._queue.put_nowait(msg) def empty(self): + """Return true if the queue is empty.""" return self._queue.empty() async def save(self) -> str: + """Convert the `MessageQueue` object to a json string.""" if self.empty(): return "[]" @@ -274,6 +281,7 @@ async def save(self) -> str: @staticmethod def load(self, v) -> "MessageQueue": + """Convert the json string to the `MessageQueue` object.""" q = MessageQueue() try: lst = json.loads(v) From d127586320d7fc1ecc81fae198dc2908cad34815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 1 Nov 2023 20:35:37 +0800 Subject: [PATCH 006/232] refactor: Refactor Message transmission & filtering --- metagpt/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 806b0e94e..1adfd525c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -235,7 +235,7 @@ def __init__(self): self._queue = Queue() def pop(self) -> Message | None: - """Pop one message from queue.""" + """Pop one message from the queue.""" try: item = self._queue.get_nowait() if item: @@ -245,7 +245,7 @@ def pop(self) -> Message | None: return None def pop_all(self) -> List[Message]: - """Pop all messages from queue.""" + """Pop all messages from the queue.""" ret = [] while True: msg = self.pop() From d685252aa0f8f1f393697b76c9e236ecc9c5117a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 10:21:15 +0800 Subject: [PATCH 007/232] feat: +unit test --- tests/metagpt/utils/test_named.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/metagpt/utils/test_named.py diff --git a/tests/metagpt/utils/test_named.py b/tests/metagpt/utils/test_named.py new file mode 100644 index 000000000..89a68b5e7 --- /dev/null +++ b/tests/metagpt/utils/test_named.py @@ -0,0 +1,21 @@ +import pytest + +from metagpt.utils.named import Named + + +@pytest.mark.asyncio +async def test_suite(): + class A(Named): + pass + + class B(A): + pass + + assert A.get_class_name() == "tests.metagpt.utils.test_named.A" + assert A().get_object_name() == "tests.metagpt.utils.test_named.A" + assert B.get_class_name() == "tests.metagpt.utils.test_named.B" + assert B().get_object_name() == "tests.metagpt.utils.test_named.B" + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From bfaeda0a90c65487b758c4c1011802e32e8f848f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 10:28:54 +0800 Subject: [PATCH 008/232] feat: +unit tests --- tests/metagpt/test_schema.py | 32 ++++++++++++++++++++++++++++--- tests/metagpt/utils/test_named.py | 7 +++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 12666e0d3..71bb39c77 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -4,18 +4,44 @@ @Time : 2023/5/20 10:40 @Author : alexanderwu @File : test_schema.py +@Modified By: mashenquan, 2023-11-1. Add `test_message`. """ +import json + +import pytest + from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage +@pytest.mark.asyncio def test_messages(): - test_content = 'test_message' + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] text = str(msgs) - roles = ['user', 'system', 'assistant', 'QA'] + roles = ["user", "system", "assistant", "QA"] assert all([i in text for i in roles]) + + +@pytest.mark.asyncio +def test_message(): + m = Message("a", role="v1") + v = m.save() + d = json.loads(v) + assert d + assert d.get("content") == "a" + assert d.get("meta_info") == {"role": "v1"} + m.set_role("v2") + v = m.save() + assert v + m = Message.load(v) + assert m.content == "a" + assert m.role == "v2" + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_named.py b/tests/metagpt/utils/test_named.py index 89a68b5e7..ff1f07205 100644 --- a/tests/metagpt/utils/test_named.py +++ b/tests/metagpt/utils/test_named.py @@ -1,3 +1,10 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023-11-1 +@Author : mashenquan +@File : test_named.py +""" import pytest from metagpt.utils.named import Named From bc1a757293b0967c3d98ad01edcf531d5c88aef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 10:40:26 +0800 Subject: [PATCH 009/232] refactor: rename async_put_message to put_message --- metagpt/environment.py | 2 +- metagpt/roles/role.py | 8 ++++---- tests/metagpt/planner/test_action_planner.py | 2 +- tests/metagpt/planner/test_basic_planner.py | 2 +- tests/metagpt/roles/test_architect.py | 2 +- tests/metagpt/roles/test_engineer.py | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index ba0645a36..7ba077080 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -50,7 +50,7 @@ def publish_message(self, message: Message): found = False for r in self.roles.values(): if message.is_recipient(r.subscribed_tags): - r.async_put_message(message) + r.put_message(message) found = True if not found: logger.warning(f"Message no recipients: {message.save()}") diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 0a6716428..6fba40bd8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -11,10 +11,10 @@ they've subscribed to through the `subscribed_tags` property. 3. Move the message receive buffer from the global variable `self._rc.env.memory` to the role's private variable `self._rc.msg_buffer` for easier message identification and asynchronous appending of messages. - 4. Standardize the way messages are passed: `publish_message` sends messages out, while `async_put_message` places + 4. Standardize the way messages are passed: `publish_message` sends messages out, while `put_message` places messages into the Role object's private message receive buffer. There are no other message transmit methods. 5. Standardize the parameters for the `run` function: the `test_message` parameter is used for testing purposes - only. In the normal workflow, you should use `publish_message` or `async_put_message` to transmit messages. + only. In the normal workflow, you should use `publish_message` or `put_message` to transmit messages. """ from __future__ import annotations @@ -239,7 +239,7 @@ def publish_message(self, msg): return self._rc.env.publish_message(msg) - def async_put_message(self, message): + def put_message(self, message): """Place the message into the Role object's private message buffer.""" if not message: return @@ -261,7 +261,7 @@ async def run(self, test_message=None): seed = test_message elif isinstance(test_message, list): seed = Message("\n".join(test_message)) - self.async_put_message(seed) + self.put_message(seed) if not await self._observe(): # If there is no new information, suspend and wait diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index a3831c08d..99cc25b72 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -26,7 +26,7 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.async_put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) + role.put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) await role._observe() await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 9efcb9367..fa7ed7074 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -29,7 +29,7 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.async_put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) + role.put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) await role._observe() await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate diff --git a/tests/metagpt/roles/test_architect.py b/tests/metagpt/roles/test_architect.py index 910c589ca..665242379 100644 --- a/tests/metagpt/roles/test_architect.py +++ b/tests/metagpt/roles/test_architect.py @@ -16,7 +16,7 @@ @pytest.mark.asyncio async def test_architect(): role = Architect() - role.async_put_message(MockMessages.req) + role.put_message(MockMessages.req) rsp = await role.run(MockMessages.prd) logger.info(rsp) assert len(rsp.content) > 0 diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index e80234b3b..93c3132ac 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -23,9 +23,9 @@ async def test_engineer(): engineer = Engineer() - engineer.async_put_message(MockMessages.req) - engineer.async_put_message(MockMessages.prd) - engineer.async_put_message(MockMessages.system_design) + engineer.put_message(MockMessages.req) + engineer.put_message(MockMessages.prd) + engineer.put_message(MockMessages.system_design) rsp = await engineer.run(MockMessages.tasks) logger.info(rsp) From 8572fa8ecd6f409f339b07db053749d2f6361c27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 10:48:45 +0800 Subject: [PATCH 010/232] feat: +unit tests --- tests/metagpt/test_schema.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 71bb39c77..06bb57a70 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -10,7 +10,7 @@ import pytest -from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage +from metagpt.schema import AIMessage, Message, Routes, SystemMessage, UserMessage @pytest.mark.asyncio @@ -42,6 +42,29 @@ def test_message(): assert m.content == "a" assert m.role == "v2" + m = Message("a", role="b", cause_by="c", x="d") + assert m.content == "a" + assert m.role == "b" + assert m.is_recipient({"c"}) + assert m.cause_by == "c" + assert m.get_meta("x") == "d" + + +@pytest.mark.asyncio +def test_routes(): + route = Routes() + route.set_from("a") + assert route.tx_from == "a" + route.add_to("b") + assert route.tx_to == {"b"} + route.add_to("c") + assert route.tx_to == {"b", "c"} + route.set_to({"e", "f"}) + assert route.tx_to == {"e", "f"} + assert route.is_recipient({"e"}) + assert route.is_recipient({"f"}) + assert not route.is_recipient({"a"}) + if __name__ == "__main__": pytest.main([__file__, "-s"]) From 660f788683b2ace2ef8ac020f044be949fd19ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 11:51:10 +0800 Subject: [PATCH 011/232] feat: + subscribe --- metagpt/roles/role.py | 4 ++ tests/metagpt/roles/test_role.py | 64 ++++++++++++++++++++++++++++++++ tests/metagpt/test_role.py | 14 ------- 3 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 tests/metagpt/roles/test_role.py delete mode 100644 tests/metagpt/test_role.py diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 6fba40bd8..318b7d7a8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -134,6 +134,10 @@ def _init_actions(self, actions): def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" tags = [get_class_name(t) for t in actions] + self.subscribe(tags) + + def subscribe(self, tags: Set[str]): + """Listen to the corresponding behaviors""" self._rc.watch.update(tags) # check RoleContext after adding watch actions self._rc.check(self._role_id) diff --git a/tests/metagpt/roles/test_role.py b/tests/metagpt/roles/test_role.py new file mode 100644 index 000000000..cefd71ada --- /dev/null +++ b/tests/metagpt/roles/test_role.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023-11-1 +@Author : mashenquan +@File : test_role.py +""" +import pytest +from pydantic import BaseModel + +from metagpt.actions import Action, ActionOutput +from metagpt.environment import Environment +from metagpt.roles import Role +from metagpt.schema import Message + + +class MockAction(Action): + async def run(self, messages, *args, **kwargs): + assert messages + return ActionOutput(content=messages[-1].content, instruct_content=messages[-1]) + + +class MockRole(Role): + def __init__(self, name="", profile="", goal="", constraints="", desc=""): + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) + self._init_actions([MockAction()]) + + +@pytest.mark.asyncio +async def test_react(): + class Input(BaseModel): + name: str + profile: str + goal: str + constraints: str + desc: str + subscription: str + + inputs = [ + { + "name": "A", + "profile": "Tester", + "goal": "Test", + "constraints": "constraints", + "desc": "desc", + "subscription": "start", + } + ] + + for i in inputs: + seed = Input(**i) + role = MockRole( + name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, desc=seed.desc + ) + role.subscribe({seed.subscription}) + env = Environment() + env.add_role(role) + env.publish_message(Message(content="test", cause_by=seed.subscription)) + while not env.is_idle: + await env.run() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py deleted file mode 100644 index 11fd804ec..000000000 --- a/tests/metagpt/test_role.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/11 14:44 -@Author : alexanderwu -@File : test_role.py -""" -from metagpt.roles import Role - - -def test_role_desc(): - i = Role(profile='Sales', desc='Best Seller') - assert i.profile == 'Sales' - assert i._setting.desc == 'Best Seller' From d5d520f6a1fac0fd512911042156215502a5d2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 11:54:14 +0800 Subject: [PATCH 012/232] feat: + subscribe --- tests/metagpt/roles/test_role.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/metagpt/roles/test_role.py b/tests/metagpt/roles/test_role.py index cefd71ada..a11e69a23 100644 --- a/tests/metagpt/roles/test_role.py +++ b/tests/metagpt/roles/test_role.py @@ -53,11 +53,17 @@ class Input(BaseModel): name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, desc=seed.desc ) role.subscribe({seed.subscription}) + assert role._rc.watch == {seed.subscription} + assert role.name == seed.name + assert role.profile == seed.profile + assert role.is_idle env = Environment() env.add_role(role) env.publish_message(Message(content="test", cause_by=seed.subscription)) + assert not role.is_idle while not env.is_idle: await env.run() + assert role.is_idle if __name__ == "__main__": From 526751073b2a48659a9959a1e365213005d2856b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 11:58:49 +0800 Subject: [PATCH 013/232] feat: + subscribe --- tests/metagpt/{roles => }/test_role.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/metagpt/{roles => }/test_role.py (100%) diff --git a/tests/metagpt/roles/test_role.py b/tests/metagpt/test_role.py similarity index 100% rename from tests/metagpt/roles/test_role.py rename to tests/metagpt/test_role.py From 834c59df19bc4aa31065268a309d913f809f0ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 12:00:45 +0800 Subject: [PATCH 014/232] feat: + subscribe --- tests/metagpt/test_role.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index a11e69a23..1b92c88cd 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -56,6 +56,9 @@ class Input(BaseModel): assert role._rc.watch == {seed.subscription} assert role.name == seed.name assert role.profile == seed.profile + assert role._setting.goal == seed.goal + assert role._setting.constraints == seed.constraints + assert role._setting.desc == seed.desc assert role.is_idle env = Environment() env.add_role(role) From bc67109fae1195debaf747beaa52b7ba452d02b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 12:02:05 +0800 Subject: [PATCH 015/232] feat: + subscribe --- tests/metagpt/test_role.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 1b92c88cd..98646041d 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -1,9 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023-11-1 -@Author : mashenquan +@Time : 2023/5/11 14:44 +@Author : alexanderwu @File : test_role.py +@Modified By: mashenquan, 2023/11/1. Add unit tests. """ import pytest from pydantic import BaseModel From 2e9a265b916fe4d1bae390195e0ccca1067c16fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 2 Nov 2023 16:27:41 +0800 Subject: [PATCH 016/232] feat: + subscribe --- metagpt/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 1adfd525c..7c84dd4bb 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -97,6 +97,7 @@ class Message(BaseModel): def __init__(self, content, **kwargs): """ + Parameters not listed below will be stored as meta info. :param content: Message content. :param instruct_content: Message content struct. :param meta_info: Message meta info. From a7632e85481550bfab4531248fa530524d9b5263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:04:25 +0800 Subject: [PATCH 017/232] refactor: update notations --- examples/agent_creator.py | 3 ++- examples/build_customized_agent.py | 3 ++- examples/debate.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/agent_creator.py b/examples/agent_creator.py index d13cbcff2..5a1398456 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -2,7 +2,8 @@ Filename: MetaGPT/examples/agent_creator.py Created Date: Tuesday, September 12th 2023, 3:28:37 pm Author: garylin2099 -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import re diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index a953dee15..af15c90ca 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -2,7 +2,8 @@ Filename: MetaGPT/examples/build_customized_agent.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import asyncio import re diff --git a/examples/debate.py b/examples/debate.py index ade1a6fc4..475d2da55 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -2,7 +2,9 @@ Filename: MetaGPT/examples/debate.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the 'cause_by' value in the 'Message' to a string, and utilize the new message distribution + feature for message filtering. """ import asyncio import platform From 93eda7f4a364fe838f1eb0839209e4aa5a49c671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:05:57 +0800 Subject: [PATCH 018/232] refactor: update notations --- examples/debate.py | 2 +- examples/sk_agent.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index 475d2da55..1f5e58839 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -3,7 +3,7 @@ Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data - type of the 'cause_by' value in the 'Message' to a string, and utilize the new message distribution + type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution feature for message filtering. """ import asyncio diff --git a/examples/sk_agent.py b/examples/sk_agent.py index 19ee53669..900696762 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -4,7 +4,8 @@ @Time : 2023/9/13 12:36 @Author : femto Zheng @File : sk_agent.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import asyncio From 96f29dadb875ba4fd5e1be06557eb3161cbb6821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:12:59 +0800 Subject: [PATCH 019/232] refactor: update notations --- metagpt/actions/action.py | 1 + metagpt/actions/write_code.py | 2 ++ metagpt/const.py | 2 ++ 3 files changed, 5 insertions(+) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 1954e750a..c6f1f1534 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : action.py +@Modified By: mashenquan, 2023-11-1. Add generic class-to-string and object-to-string conversion functionality. """ import re from abc import ABC diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 421211d60..f0ef2b6d6 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from tenacity import retry, stop_after_attempt, wait_fixed diff --git a/metagpt/const.py b/metagpt/const.py index 3fbc26784..e783ec8d0 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,6 +4,8 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, added key definitions for + common properties in the Message. """ from pathlib import Path From e49f8a010e7ea9797ae25c6d1b61c33f26373a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:15:01 +0800 Subject: [PATCH 020/232] refactor: update notations --- metagpt/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index 7ba077080..028e98e8e 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -4,7 +4,7 @@ @Time : 2023/5/11 22:12 @Author : alexanderwu @File : environment.py -@Modified By: mashenquan, 2023-11-1. Optimization: +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116: 1. Remove the functionality of `Environment` class as a public message buffer. 2. Standardize the message forwarding behavior of the `Environment` class. 3. Add the `is_idle` property. From 67f07b66cda2598d4e0887e95cd8d6099a6d6336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:26:33 +0800 Subject: [PATCH 021/232] refactor: update notations --- metagpt/memory/longterm_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index b5bb73b6b..e73ae334e 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ @Desc : the implement of Long-term memory -@Modified By: mashenquan, 2023-11-1. Optimization: +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: 1. Replace code related to message filtering with the `Message.is_recipient` function. """ From ddd2d40ff3c5bed217918e64d346ce0ce7fa5f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:29:37 +0800 Subject: [PATCH 022/232] refactor: update notations --- metagpt/memory/memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 8e01544f1..282e89b17 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,7 +4,8 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py -@Modified By: mashenquan, 2023-11-1. Standardize the design of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: + Modify the new message distribution feature for message filtering. """ from collections import defaultdict from typing import Iterable, Set From a996440d5e7bbfd6af24ed026a3bc332f6856e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:36:27 +0800 Subject: [PATCH 023/232] refactor: update notations --- metagpt/memory/memory.py | 2 +- metagpt/roles/engineer.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 282e89b17..7f04be63d 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : memory.py @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: - Modify the new message distribution feature for message filtering. + Updated the message filtering logic. """ from collections import defaultdict from typing import Iterable, Set diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 9826ea0b7..ff71a61d8 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -4,10 +4,12 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : engineer.py -@Modified By: mashenquan, 2023-11-1. Optimization: - 1. Consolidate message reception and processing logic within `_observe`. - 2. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. - 3. Supplemented the external transmission of internal messages. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116: + 1. Modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message + distribution feature for message filtering. + 2. Consolidate message reception and processing logic within `_observe`. + 3. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. + 4. Supplemented the external transmission of internal messages. """ import asyncio import shutil From 6bd9a76997b9be323c539d0a6c34ac4658df49b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:43:40 +0800 Subject: [PATCH 024/232] refactor: update notations --- metagpt/roles/qa_engineer.py | 3 ++- metagpt/roles/researcher.py | 3 ++- metagpt/roles/role.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index b83ab6e21..5cc35a878 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -4,7 +4,8 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : qa_engineer.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. """ import os from pathlib import Path diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 6e89b9fe7..4ec6f31e1 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """ -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 318b7d7a8..79a9fb2de 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,7 +4,7 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py -@Modified By: mashenquan, 2023-11-1. Optimization: +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: 1. Merge the `recv` functionality into the `_observe` function. Future message reading operations will be consolidated within the `_observe` function. 2. Standardize the message filtering for string label matching. Role objects can access the message labels From 17c5f80d809085cd324261b9661fc06089940780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:48:47 +0800 Subject: [PATCH 025/232] refactor: update notations --- metagpt/roles/seacher.py | 3 ++- metagpt/roles/sk_agent.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index 95be89277..d0b841f39 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -4,7 +4,8 @@ @Time : 2023/5/23 17:25 @Author : alexanderwu @File : seacher.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from metagpt.actions import ActionOutput, SearchAndSummarize from metagpt.logs import logger diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index abebb9605..5b8d333bd 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -4,7 +4,9 @@ @Time : 2023/9/13 12:23 @Author : femto Zheng @File : sk_agent.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution + feature for message filtering. """ from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner From 953a003e1e57b2dbf741b53e0a7cdee344bae593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:53:47 +0800 Subject: [PATCH 026/232] refactor: update notations --- metagpt/schema.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 7c84dd4bb..34e6fa07b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -4,7 +4,8 @@ @Time : 2023/5/8 22:12 @Author : alexanderwu @File : schema.py -@Modified By: mashenquan, 2023-10-31, optimize class members. +@Modified By: mashenquan, 2023-10-31. According to Chapter 2.2.1 of RFC 116: + Replanned the distribution of responsibilities and functional positioning of `Message` class attributes. """ from __future__ import annotations @@ -97,7 +98,7 @@ class Message(BaseModel): def __init__(self, content, **kwargs): """ - Parameters not listed below will be stored as meta info. + Parameters not listed below will be stored as meta info, including custom parameters. :param content: Message content. :param instruct_content: Message content struct. :param meta_info: Message meta info. From b1386a01f5ce016268902f4fc82845079f8d089b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 11:57:10 +0800 Subject: [PATCH 027/232] refactor: update notations --- metagpt/software_company.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 4bedec0e1..d29d8926d 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,9 +4,10 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py -@Modified By: mashenquan, 2023-11-1. Optimization: - 1. Standardize the design of message filtering-related features. - 2. Abandon the design of having `Environment` store all messages. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: + 1. Change the data type of the `cause_by` value in the `Message` to a string to support the new message + distribution feature. + 2. Abandon the design of having `Environment` store all messages. """ from pydantic import BaseModel, Field From 290479969b0c0386b8004cd46d78b22f603aa805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 12:00:18 +0800 Subject: [PATCH 028/232] refactor: update notations --- metagpt/utils/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index df4688378..219ed9f04 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -4,6 +4,8 @@ @Time : 2023/4/29 16:07 @Author : alexanderwu @File : common.py +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116: + Add generic class-to-string and object-to-string conversion functionality. """ import ast import contextlib From bdf59b67bd4bfe5b381b9c61cf59086af01127c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 12:02:52 +0800 Subject: [PATCH 029/232] refactor: update notations --- tests/metagpt/actions/test_write_prd.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 40ab20dad..0da7831c6 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -4,7 +4,8 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_prd.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import pytest @@ -18,7 +19,7 @@ async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement.get_class_name())) + prd = await product_manager.run(Message(content=requirements, cause_by=BossRequirement.get_class_name())) logger.info(requirements) logger.info(prd) From 4b6745baaa26b78c2d0c7fcff70079f87674f35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 12:04:00 +0800 Subject: [PATCH 030/232] refactor: update notations --- tests/metagpt/memory/test_longterm_memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index c40d7ab9d..712402db1 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- """ @Desc : unittest of `metagpt/memory/longterm_memory.py` -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from metagpt.actions import BossRequirement From 2c551c1fd38ad0bf318591bc92f5075f8aa6eead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 12:05:11 +0800 Subject: [PATCH 031/232] refactor: update notations --- tests/metagpt/memory/test_memory_storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 881b47d6f..c9585054a 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- """ @Desc : the unittests of metagpt/memory/memory_storage.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ From 7e71ad85ca2dd638a044f9130737bb3685e7089d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 13:29:05 +0800 Subject: [PATCH 032/232] refactor: update notations --- tests/metagpt/planner/test_action_planner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index 99cc25b72..f0a18da46 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -4,9 +4,9 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py -@Modified By: mashenquan, 2023-11-1. Optimization: - 1. Standardize the usage of message filtering-related features. - 2. Standardize the usage of message transmission. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution + feature for message handling. """ import pytest from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill From 55fe826b06f40298fc46b46bd56d43fe5a580536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 13:29:35 +0800 Subject: [PATCH 033/232] refactor: update notations --- tests/metagpt/planner/test_basic_planner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index fa7ed7074..7623aee95 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -4,9 +4,9 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py -@Modified By: mashenquan, 2023-11-1. Optimization: - 1. Standardize the usage of message filtering-related features. - 2. Standardize the usage of message transmission. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution + feature for message handling. """ import pytest from semantic_kernel.core_skills import TextSkill From 78f3f128c046c93c29a906761fcbc4de03e88a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 13:40:48 +0800 Subject: [PATCH 034/232] refactor: update notations --- tests/metagpt/roles/mock.py | 3 ++- tests/metagpt/roles/test_architect.py | 4 +++- tests/metagpt/roles/test_engineer.py | 4 +++- tests/metagpt/test_environment.py | 3 ++- tests/metagpt/test_message.py | 1 + tests/metagpt/test_role.py | 3 ++- tests/metagpt/test_schema.py | 3 ++- tests/metagpt/utils/test_serialize.py | 3 ++- 8 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index b9891cd81..e67d64abc 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -4,7 +4,8 @@ @Time : 2023/5/12 13:05 @Author : alexanderwu @File : mock.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message diff --git a/tests/metagpt/roles/test_architect.py b/tests/metagpt/roles/test_architect.py index 665242379..4effadaaa 100644 --- a/tests/metagpt/roles/test_architect.py +++ b/tests/metagpt/roles/test_architect.py @@ -4,7 +4,9 @@ @Time : 2023/5/20 14:37 @Author : alexanderwu @File : test_architect.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message transmission. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution + feature for message handling. """ import pytest diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index 93c3132ac..93f2efb77 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -4,7 +4,9 @@ @Time : 2023/5/12 10:14 @Author : alexanderwu @File : test_engineer.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message transmission. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data + type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution + feature for message handling. """ import pytest diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index 755798b17..714618852 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -4,7 +4,8 @@ @Time : 2023/5/12 00:47 @Author : alexanderwu @File : test_environment.py -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message transmission. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import pytest diff --git a/tests/metagpt/test_message.py b/tests/metagpt/test_message.py index 4f46311ce..04d85d9e4 100644 --- a/tests/metagpt/test_message.py +++ b/tests/metagpt/test_message.py @@ -4,6 +4,7 @@ @Time : 2023/5/16 10:57 @Author : alexanderwu @File : test_message.py +@Modified By: mashenquan, 2023-11-1. Modify coding style. """ import pytest diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 98646041d..f0ef4b3d9 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -4,7 +4,8 @@ @Time : 2023/5/11 14:44 @Author : alexanderwu @File : test_role.py -@Modified By: mashenquan, 2023/11/1. Add unit tests. +@Modified By: mashenquan, 2023-11-1. In line with Chapter 2.2.1 and 2.2.2 of RFC 116, introduce unit tests for + the utilization of the new message distribution feature in message handling. """ import pytest from pydantic import BaseModel diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 06bb57a70..2fa76fcad 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -4,7 +4,8 @@ @Time : 2023/5/20 10:40 @Author : alexanderwu @File : test_schema.py -@Modified By: mashenquan, 2023-11-1. Add `test_message`. +@Modified By: mashenquan, 2023-11-1. In line with Chapter 2.2.1 and 2.2.2 of RFC 116, introduce unit tests for + the utilization of the new feature of `Message` class. """ import json diff --git a/tests/metagpt/utils/test_serialize.py b/tests/metagpt/utils/test_serialize.py index 5a0840c87..7889f96fe 100644 --- a/tests/metagpt/utils/test_serialize.py +++ b/tests/metagpt/utils/test_serialize.py @@ -2,7 +2,8 @@ # -*- coding: utf-8 -*- """ @Desc : the unittest of serialize -@Modified By: mashenquan, 2023-11-1. Standardize the usage of message filtering-related features. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of + the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from typing import List, Tuple From e667fb4f00e19c3c8e36c51793c20dbcedf662dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 13:53:12 +0800 Subject: [PATCH 035/232] refactor: update notations --- metagpt/roles/role.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 79a9fb2de..753c22134 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -258,14 +258,14 @@ async def _react(self) -> Message: async def run(self, test_message=None): """Observe, and think and act based on the results of the observation""" if test_message: # For test - seed = None + msg = None if isinstance(test_message, str): - seed = Message(test_message) + msg = Message(test_message) elif isinstance(test_message, Message): - seed = test_message + msg = test_message elif isinstance(test_message, list): - seed = Message("\n".join(test_message)) - self.put_message(seed) + msg = Message("\n".join(test_message)) + self.put_message(msg) if not await self._observe(): # If there is no new information, suspend and wait From 532099a7c6c7ebe5e20a657067e3a8540e7a068f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 15:14:05 +0800 Subject: [PATCH 036/232] refactor: update notations --- metagpt/environment.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index 028e98e8e..b93eeb6b2 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -44,8 +44,15 @@ def add_roles(self, roles: Iterable[Role]): for role in roles: self.add_role(role) - def publish_message(self, message: Message): - """Distribute the message to the recipients.""" + def publish_message(self, message: Message) -> bool: + """ + Distribute the message to the recipients. + In accordance with the Message routing structure design in Chapter 2.2.1 of RFC 116, as already planned + in RFC 113 for the entire system, the routing information in the Message is only responsible for + specifying the message recipient, without concern for where the message recipient is located. How to + route the message to the message recipient is a problem addressed by the transport framework designed + in RFC 113. + """ logger.info(f"publish_message: {message.save()}") found = False for r in self.roles.values(): @@ -55,6 +62,12 @@ def publish_message(self, message: Message): if not found: logger.warning(f"Message no recipients: {message.save()}") + # Implemented the functionality related to remote message forwarding as described in RFC 113. Awaiting release. + # if self._parent: + # return self._parent.publish_message(message) + + return True + async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once From 8137e1af5018169542055f064c1a8ef9b4333dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 3 Nov 2023 18:08:57 +0800 Subject: [PATCH 037/232] fixbug: creation of separate indices for each label --- metagpt/memory/memory.py | 6 ++++-- tests/metagpt/test_role.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 7f04be63d..cf3140bdb 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -26,8 +26,10 @@ def add(self, message: Message): if message in self.storage: return self.storage.append(message) - if message.cause_by: - self.index[message.cause_by].append(message) + # According to the design of RFC 116, it allows message filtering based on different labels, thus + # necessitating the creation of separate indices for each label. + for k in message.tx_to: + self.index[k].append(message) def add_batch(self, messages: Iterable[Message]): for message in messages: diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index f0ef4b3d9..829f75bc5 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -64,6 +64,11 @@ class Input(BaseModel): assert role.is_idle env = Environment() env.add_role(role) + env.publish_message(Message(content="test", tx_to=seed.subscription)) + assert not role.is_idle + while not env.is_idle: + await env.run() + assert role.is_idle env.publish_message(Message(content="test", cause_by=seed.subscription)) assert not role.is_idle while not env.is_idle: From 2688fe680adb60e355f7176d439df31b28237db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 14:07:33 +0800 Subject: [PATCH 038/232] feat: According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing functionality is to be consolidated into the Environment class. --- metagpt/environment.py | 26 ++++++++++++++++++-------- metagpt/roles/role.py | 9 +++++++-- tests/metagpt/test_role.py | 8 ++++++++ 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index b93eeb6b2..0fa330a83 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -8,9 +8,11 @@ 1. Remove the functionality of `Environment` class as a public message buffer. 2. Standardize the message forwarding behavior of the `Environment` class. 3. Add the `is_idle` property. +@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing + functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable +from typing import Iterable, Set from pydantic import BaseModel, Field @@ -26,6 +28,7 @@ class Environment(BaseModel): """ roles: dict[str, Role] = Field(default_factory=dict) + consumers: dict[Role, Set] = Field(default_factory=dict) class Config: arbitrary_types_allowed = True @@ -36,6 +39,8 @@ def add_role(self, role: Role): """ role.set_env(self) self.roles[role.profile] = role + # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 + self.set_subscribed_tags(role, role.subscribed_tags) def add_roles(self, roles: Iterable[Role]): """增加一批在当前环境的角色 @@ -55,17 +60,14 @@ def publish_message(self, message: Message) -> bool: """ logger.info(f"publish_message: {message.save()}") found = False - for r in self.roles.values(): - if message.is_recipient(r.subscribed_tags): - r.put_message(message) + # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 + for obj, subscribed_tags in self.consumers.items(): + if message.is_recipient(subscribed_tags): + obj.put_message(message) found = True if not found: logger.warning(f"Message no recipients: {message.save()}") - # Implemented the functionality related to remote message forwarding as described in RFC 113. Awaiting release. - # if self._parent: - # return self._parent.publish_message(message) - return True async def run(self, k=1): @@ -100,3 +102,11 @@ def is_idle(self): if not r.is_idle: return False return True + + def get_subscribed_tags(self, obj): + """Get the labels for messages to be consumed by the object.""" + return self.consumers.get(obj, {}) + + def set_subscribed_tags(self, obj, tags): + """Set the labels for message to be consumed by the object""" + self.consumers[obj] = tags diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 753c22134..eacaa0034 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -15,6 +15,8 @@ messages into the Role object's private message receive buffer. There are no other message transmit methods. 5. Standardize the parameters for the `run` function: the `test_message` parameter is used for testing purposes only. In the normal workflow, you should use `publish_message` or `put_message` to transmit messages. +@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing + functionality is to be consolidated into the `Environment` class. """ from __future__ import annotations @@ -133,7 +135,7 @@ def _init_actions(self, actions): def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" - tags = [get_class_name(t) for t in actions] + tags = {get_class_name(t) for t in actions} self.subscribe(tags) def subscribe(self, tags: Set[str]): @@ -141,6 +143,8 @@ def subscribe(self, tags: Set[str]): self._rc.watch.update(tags) # check RoleContext after adding watch actions self._rc.check(self._role_id) + if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 + self._rc.env.set_subscribed_tags(self, self.subscribed_tags) def _set_state(self, state): """Update the current state.""" @@ -149,7 +153,8 @@ def _set_state(self, state): self._rc.todo = self._actions[self._rc.state] def set_env(self, env: "Environment"): - """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" + """Set the environment in which the role works. The role can talk to the environment and can also receive + messages by observing.""" self._rc.env = env @property diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 829f75bc5..7794c9b57 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -6,7 +6,11 @@ @File : test_role.py @Modified By: mashenquan, 2023-11-1. In line with Chapter 2.2.1 and 2.2.2 of RFC 116, introduce unit tests for the utilization of the new message distribution feature in message handling. +@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing + functionality is to be consolidated into the `Environment` class. """ +import uuid + import pytest from pydantic import BaseModel @@ -64,6 +68,7 @@ class Input(BaseModel): assert role.is_idle env = Environment() env.add_role(role) + assert env.get_subscribed_tags(role) == {seed.subscription} env.publish_message(Message(content="test", tx_to=seed.subscription)) assert not role.is_idle while not env.is_idle: @@ -74,6 +79,9 @@ class Input(BaseModel): while not env.is_idle: await env.run() assert role.is_idle + tag = uuid.uuid4().hex + role.subscribe({tag}) + assert env.get_subscribed_tags(role) == {seed.subscription, tag} if __name__ == "__main__": From c4eb028a8303a2dfb9fbb8018d751e5343c01d91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 14:26:48 +0800 Subject: [PATCH 039/232] refactor: save -> dump --- metagpt/environment.py | 4 ++-- metagpt/schema.py | 14 +++++++------- tests/metagpt/test_schema.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index 0fa330a83..a7e6322ff 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -58,7 +58,7 @@ def publish_message(self, message: Message) -> bool: route the message to the message recipient is a problem addressed by the transport framework designed in RFC 113. """ - logger.info(f"publish_message: {message.save()}") + logger.info(f"publish_message: {message.dump()}") found = False # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 for obj, subscribed_tags in self.consumers.items(): @@ -66,7 +66,7 @@ def publish_message(self, message: Message) -> bool: obj.put_message(message) found = True if not found: - logger.warning(f"Message no recipients: {message.save()}") + logger.warning(f"Message no recipients: {message.dump()}") return True diff --git a/metagpt/schema.py b/metagpt/schema.py index 34e6fa07b..bb8d8b42c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -188,7 +188,7 @@ def to_dict(self) -> dict: """Return a dict containing `role` and `content` for the LLM call.l""" return {"role": self.role, "content": self.content} - def save(self) -> str: + def dump(self) -> str: """Convert the object to json string""" return self.json(exclude_none=True) @@ -264,7 +264,7 @@ def empty(self): """Return true if the queue is empty.""" return self._queue.empty() - async def save(self) -> str: + async def dump(self) -> str: """Convert the `MessageQueue` object to a json string.""" if self.empty(): return "[]" @@ -299,7 +299,7 @@ def load(self, v) -> "MessageQueue": if __name__ == "__main__": m = Message("a", role="v1") m.set_role("v2") - v = m.save() + v = m.dump() m = Message.load(v) test_content = "test_message" @@ -312,9 +312,9 @@ def load(self, v) -> "MessageQueue": logger.info(msgs) jsons = [ - UserMessage(test_content).save(), - SystemMessage(test_content).save(), - AIMessage(test_content).save(), - Message(test_content, role="QA").save(), + UserMessage(test_content).dump(), + SystemMessage(test_content).dump(), + AIMessage(test_content).dump(), + Message(test_content, role="QA").dump(), ] logger.info(jsons) diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 2fa76fcad..21ba3fd14 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -31,13 +31,13 @@ def test_messages(): @pytest.mark.asyncio def test_message(): m = Message("a", role="v1") - v = m.save() + v = m.dump() d = json.loads(v) assert d assert d.get("content") == "a" assert d.get("meta_info") == {"role": "v1"} m.set_role("v2") - v = m.save() + v = m.dump() assert v m = Message.load(v) assert m.content == "a" From 1febf168e7bd7e2e10becbdad14ed42d03f2b443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 16:20:47 +0800 Subject: [PATCH 040/232] refactor: Override cause_by --- metagpt/schema.py | 33 +++++++++++++++++++++++++++++++++ tests/metagpt/test_schema.py | 10 ++++++++++ 2 files changed, 43 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index bb8d8b42c..52020c468 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -24,6 +24,7 @@ MESSAGE_ROUTE_TO, ) from metagpt.logs import logger +from metagpt.utils.common import get_class_name, get_object_name class RawMessage(TypedDict): @@ -87,6 +88,14 @@ def tx_to(self): route = self._get_route() return route.get(MESSAGE_ROUTE_TO) + def replace(self, old_val, new_val): + """Replace old value with new value""" + route = self._get_route() + tags = route.get(MESSAGE_ROUTE_TO, set()) + tags.discard(old_val) + tags.add(new_val) + route[MESSAGE_ROUTE_TO] = tags + class Message(BaseModel): """list[: ]""" @@ -147,6 +156,26 @@ def cause_by(self): """Labels for the consumer to filter its subscribed messages, also serving as meta info.""" return self.get_meta(MESSAGE_ROUTE_CAUSE_BY) + def __setattr__(self, key, val): + """Override `@property.setter`""" + if key == MESSAGE_ROUTE_CAUSE_BY: + self.set_cause_by(val) + return + super().__setattr__(key, val) + + def set_cause_by(self, val): + """Update the value of `cause_by` in the `meta_info` and `routes` attributes.""" + old_value = self.get_meta(MESSAGE_ROUTE_CAUSE_BY) + new_value = None + if isinstance(val, str): + new_value = val + elif not callable(val): + new_value = get_object_name(val) + else: + new_value = get_class_name(val) + self.set_meta(MESSAGE_ROUTE_CAUSE_BY, new_value) + self.route.replace(old_value, new_value) + @property def tx_from(self): """Message route info tells who sent this message.""" @@ -301,6 +330,10 @@ def load(self, v) -> "MessageQueue": m.set_role("v2") v = m.dump() m = Message.load(v) + m.cause_by = "Message" + m.cause_by = Routes + m.cause_by = Routes() + m.content = "b" test_content = "test_message" msgs = [ diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 21ba3fd14..e4aa0c0dd 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -11,6 +11,7 @@ import pytest +from metagpt.actions import Action from metagpt.schema import AIMessage, Message, Routes, SystemMessage, UserMessage @@ -50,6 +51,15 @@ def test_message(): assert m.cause_by == "c" assert m.get_meta("x") == "d" + m.cause_by = "Message" + assert m.cause_by == "Message" + m.cause_by = Action + assert m.cause_by == Action.get_class_name() + m.cause_by = Action() + assert m.cause_by == Action.get_class_name() + m.content = "b" + assert m.content == "b" + @pytest.mark.asyncio def test_routes(): From d9775037b68eee015f372e27e664e6f5952e9f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 16:46:32 +0800 Subject: [PATCH 041/232] refactor: @cause_by.setter --- examples/debate.py | 10 +++---- examples/sk_agent.py | 8 +++--- metagpt/actions/action.py | 3 +-- metagpt/actions/write_code.py | 2 +- metagpt/roles/engineer.py | 12 ++++----- metagpt/roles/qa_engineer.py | 20 +++++++------- metagpt/roles/role.py | 7 +++-- metagpt/schema.py | 19 ++++++------- metagpt/software_company.py | 7 +++-- metagpt/utils/common.py | 10 +++++++ metagpt/utils/named.py | 21 --------------- tests/metagpt/actions/test_write_prd.py | 2 +- tests/metagpt/memory/test_longterm_memory.py | 10 +++---- tests/metagpt/memory/test_memory_storage.py | 16 +++++------ tests/metagpt/planner/test_action_planner.py | 2 +- tests/metagpt/planner/test_basic_planner.py | 2 +- tests/metagpt/roles/mock.py | 8 +++--- tests/metagpt/test_environment.py | 2 +- tests/metagpt/test_schema.py | 5 ++-- tests/metagpt/utils/test_named.py | 28 -------------------- tests/metagpt/utils/test_serialize.py | 2 +- 21 files changed, 73 insertions(+), 123 deletions(-) delete mode 100644 metagpt/utils/named.py delete mode 100644 tests/metagpt/utils/test_named.py diff --git a/examples/debate.py b/examples/debate.py index 1f5e58839..c1d997678 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -66,7 +66,7 @@ async def _observe(self) -> int: async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions([ShoutOut.get_class_name()]) + msg_history = self._rc.memory.get_by_actions([ShoutOut]) context = [] for m in msg_history: context.append(str(m)) @@ -77,7 +77,7 @@ async def _act(self) -> Message: msg = Message( content=rsp, role=self.profile, - cause_by=ShoutOut.get_class_name(), + cause_by=ShoutOut, tx_from=self.name, tx_to=self.opponent_name, ) @@ -102,14 +102,14 @@ async def _observe(self) -> int: await super()._observe() # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, # disregard own messages from the last round - message_filter = {BossRequirement.get_class_name(), self.name} + message_filter = {BossRequirement, self.name} self._rc.news = [msg for msg in self._rc.news if msg.is_recipient(message_filter)] return len(self._rc.news) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions([BossRequirement.get_class_name(), ShoutOut.get_class_name()]) + msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) context = [] for m in msg_history: context.append(str(m)) @@ -120,7 +120,7 @@ async def _act(self) -> Message: msg = Message( content=rsp, role=self.profile, - cause_by=ShoutOut.get_class_name(), + cause_by=ShoutOut, tx_from=self.name, tx_to=self.opponent_name, ) diff --git a/examples/sk_agent.py b/examples/sk_agent.py index 900696762..21714cca1 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -41,7 +41,7 @@ async def basic_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) + await role.run(Message(content=task, cause_by=BossRequirement)) async def sequential_planner_example(): @@ -55,7 +55,7 @@ async def sequential_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) + await role.run(Message(content=task, cause_by=BossRequirement)) async def basic_planner_web_search_example(): @@ -66,7 +66,7 @@ async def basic_planner_web_search_example(): role.import_skill(SkSearchEngine(), "WebSearchSkill") # role.import_semantic_skill_from_directory(skills_directory, "QASkill") - await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) + await role.run(Message(content=task, cause_by=BossRequirement)) async def action_planner_example(): @@ -77,7 +77,7 @@ async def action_planner_example(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - await role.run(Message(content=task, cause_by=BossRequirement.get_class_name())) # it will choose mathskill.Add + await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add if __name__ == "__main__": diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index c6f1f1534..fd114b332 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -17,10 +17,9 @@ from metagpt.logs import logger from metagpt.utils.common import OutputParser from metagpt.utils.custom_decoder import CustomDecoder -from metagpt.utils.named import Named -class Action(ABC, Named): +class Action(ABC): def __init__(self, name: str = "", context=None, llm: LLM = None): self.name: str = name if llm is None: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index f0ef2b6d6..8b6451134 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -58,7 +58,7 @@ def _save(self, context, filename, code): if self._is_invalid(filename): return - message_filter = {WriteDesign.get_class_name()} + message_filter = {WriteDesign} design = [i for i in context if i.is_recipient(message_filter)][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index ff71a61d8..7f05c52c5 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -102,7 +102,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign.get_class_name())[-1] + msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -130,7 +130,7 @@ async def _act_mp(self) -> Message: todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks.get_class_name(), WriteDesign.get_class_name()]), + context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo, ) todo_coros.append(todo_coro) @@ -185,7 +185,7 @@ async def _act_sp_precision(self) -> Message: TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ context = [] - msg_filters = [WriteDesign.get_class_name(), WriteTasks.get_class_name(), WriteCode.get_class_name()] + msg_filters = [WriteDesign, WriteTasks, WriteCode] msg = self._rc.memory.get_by_actions(msg_filters) for m in msg: context.append(m.content) @@ -201,7 +201,7 @@ async def _act_sp_precision(self) -> Message: logger.error("code review failed!", e) pass file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode.get_class_name()) + msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) self.publish_message(msg) @@ -231,7 +231,7 @@ async def _observe(self) -> int: return ret # Parse task lists - message_filter = {WriteTasks.get_class_name()} + message_filter = {WriteTasks} for message in self._rc.news: if not message.is_recipient(message_filter): continue @@ -241,7 +241,7 @@ async def _observe(self) -> int: async def _think(self) -> None: # In asynchronous scenarios, first check if the required messages are ready. - filters = {WriteTasks.get_class_name()} + filters = {WriteTasks} msgs = self._rc.memory.get_by_actions(filters) if not msgs: self._rc.todo = None diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 5cc35a878..64d7f9702 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -50,7 +50,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign.get_class_name())[-1] + msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -99,7 +99,7 @@ async def _write_test(self, message: Message) -> None: msg = Message( content=str(file_info), role=self.profile, - cause_by=WriteTest.get_class_name(), + cause_by=WriteTest, tx_from=self.profile, tx_to=self.profile, ) @@ -133,9 +133,7 @@ async def _run_code(self, msg): recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message( - content=content, role=self.profile, cause_by=RunCode.get_class_name(), tx_from=self.profile, tx_to=recipient - ) + msg = Message(content=content, role=self.profile, cause_by=RunCode, tx_from=self.profile, tx_to=recipient) self.publish_message(msg) async def _debug_error(self, msg): @@ -147,7 +145,7 @@ async def _debug_error(self, msg): msg = Message( content=file_info, role=self.profile, - cause_by=DebugError.get_class_name(), + cause_by=DebugError, tx_from=self.profile, tx_to=recipient, ) @@ -165,14 +163,14 @@ async def _act(self) -> Message: result_msg = Message( content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", role=self.profile, - cause_by=WriteTest.get_class_name(), + cause_by=WriteTest, tx_from=self.profile, ) return result_msg - code_filters = {WriteCode.get_class_name(), WriteCodeReview.get_class_name()} - test_filters = {WriteTest.get_class_name(), DebugError.get_class_name()} - run_filters = {RunCode.get_class_name()} + code_filters = {WriteCode, WriteCodeReview} + test_filters = {WriteTest, DebugError} + run_filters = {RunCode} for msg in self._rc.news: # Decide what to do based on observed msg type, currently defined by human, # might potentially be moved to _think, that is, let the agent decides for itself @@ -189,7 +187,7 @@ async def _act(self) -> Message: result_msg = Message( content=f"Round {self.test_round} of tests done", role=self.profile, - cause_by=WriteTest.get_class_name(), + cause_by=WriteTest, tx_from=self.profile, ) return result_msg diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index eacaa0034..87a03b391 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -31,7 +31,6 @@ from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message, MessageQueue from metagpt.utils.common import get_class_name, get_object_name -from metagpt.utils.named import Named PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -107,7 +106,7 @@ def history(self) -> list[Message]: return self.memory.get() -class Role(Named): +class Role: """Role/Agent""" def __init__(self, name="", profile="", goal="", constraints="", desc=""): @@ -174,10 +173,10 @@ def subscribed_tags(self) -> Set: return self._rc.watch return { self.name, - self.get_object_name(), + get_object_name(self), self.profile, f"{self.name}({self.profile})", - f"{self.name}({self.get_object_name()})", + f"{self.name}({get_object_name(self)})", } def _get_prefix(self): diff --git a/metagpt/schema.py b/metagpt/schema.py index 52020c468..1082c5ddb 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -24,7 +24,7 @@ MESSAGE_ROUTE_TO, ) from metagpt.logs import logger -from metagpt.utils.common import get_class_name, get_object_name +from metagpt.utils.common import any_to_str class RawMessage(TypedDict): @@ -129,11 +129,12 @@ def __init__(self, content, **kwargs): if k in attribute_names: continue if k == MESSAGE_ROUTE_FROM: - self.set_from(v) + self.set_from(any_to_str(v)) continue if k == MESSAGE_ROUTE_CAUSE_BY: - self.meta_info[k] = v - if k == MESSAGE_ROUTE_TO or k == MESSAGE_ROUTE_CAUSE_BY: + self.set_cause_by(v) + continue + if k == MESSAGE_ROUTE_TO: self.add_to(v) continue self.meta_info[k] = v @@ -161,18 +162,14 @@ def __setattr__(self, key, val): if key == MESSAGE_ROUTE_CAUSE_BY: self.set_cause_by(val) return + if key == MESSAGE_ROUTE_FROM: + self.set_from(any_to_str(val)) super().__setattr__(key, val) def set_cause_by(self, val): """Update the value of `cause_by` in the `meta_info` and `routes` attributes.""" old_value = self.get_meta(MESSAGE_ROUTE_CAUSE_BY) - new_value = None - if isinstance(val, str): - new_value = val - elif not callable(val): - new_value = get_object_name(val) - else: - new_value = get_class_name(val) + new_value = any_to_str(val) self.set_meta(MESSAGE_ROUTE_CAUSE_BY, new_value) self.route.replace(old_value, new_value) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index d29d8926d..57bd5db19 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -18,10 +18,9 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException -from metagpt.utils.named import Named -class SoftwareCompany(BaseModel, Named): +class SoftwareCompany(BaseModel): """ Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to writing executable code. @@ -55,8 +54,8 @@ def start_project(self, idea): Message( role="BOSS", content=idea, - cause_by=BossRequirement.get_class_name(), - tx_from=SoftwareCompany.get_class_name(), + cause_by=BossRequirement, + tx_from=SoftwareCompany, ) ) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 219ed9f04..b372f0d8d 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -315,3 +315,13 @@ def get_object_name(obj) -> str: """Return class name of the object""" cls = type(obj) return f"{cls.__module__}.{cls.__name__}" + + +def any_to_str(val) -> str: + """Return the class name or the class name of the object, or 'val' if it's a string type.""" + if isinstance(val, str): + return val + if not callable(val): + return get_object_name(val) + + return get_class_name(val) diff --git a/metagpt/utils/named.py b/metagpt/utils/named.py deleted file mode 100644 index e4da574e8..000000000 --- a/metagpt/utils/named.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/11/1 -@Author : mashenquan -@File : named.py -""" - - -class Named: - """A base class with functions for converting classes to names and objects to class names.""" - - @classmethod - def get_class_name(cls): - """Return class name""" - return f"{cls.__module__}.{cls.__name__}" - - def get_object_name(self): - """Return class name of the object""" - cls = type(self) - return f"{cls.__module__}.{cls.__name__}" diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 0da7831c6..5a121adce 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -19,7 +19,7 @@ async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.run(Message(content=requirements, cause_by=BossRequirement.get_class_name())) + prd = await product_manager.run(Message(content=requirements, cause_by=BossRequirement)) logger.info(requirements) logger.info(prd) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index 712402db1..b33dd312d 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -19,24 +19,24 @@ def test_ltm_search(): assert len(openai_api_key) > 20 role_id = "UTUserLtm(Product Manager)" - rc = RoleContext(watch=[BossRequirement.get_class_name()]) + rc = RoleContext(watch=[BossRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) idea = "Write a cli snake game" - message = Message(role="BOSS", content=idea, cause_by=BossRequirement.get_class_name()) + message = Message(role="BOSS", content=idea, cause_by=BossRequirement) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) sim_idea = "Write a game of cli snake" - sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement.get_class_name()) + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) new_idea = "Write a 2048 web game" - new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement.get_class_name()) + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -52,7 +52,7 @@ def test_ltm_search(): assert len(news) == 0 new_idea = "Write a Battle City" - new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement.get_class_name()) + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index c9585054a..c40bbbba5 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -18,7 +18,7 @@ def test_idea_message(): idea = "Write a cli snake game" role_id = "UTUser1(Product Manager)" - message = Message(role="BOSS", content=idea, cause_by=BossRequirement.get_class_name()) + message = Message(role="BOSS", content=idea, cause_by=BossRequirement) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -28,12 +28,12 @@ def test_idea_message(): assert memory_storage.is_initialized is True sim_idea = "Write a game of cli snake" - sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement.get_class_name()) + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement) new_messages = memory_storage.search(sim_message) assert len(new_messages) == 0 # similar, return [] new_idea = "Write a 2048 web game" - new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement.get_class_name()) + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -49,7 +49,7 @@ def test_actionout_message(): role_id = "UTUser2(Architect)" content = "The boss has requested the creation of a command-line interface (CLI) snake game" message = Message( - content=content, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() + content=content, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD ) # WritePRD as test action memory_storage: MemoryStorage = MemoryStorage() @@ -60,16 +60,12 @@ def test_actionout_message(): assert memory_storage.is_initialized is True sim_conent = "The request is command-line interface (CLI) snake game" - sim_message = Message( - content=sim_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() - ) + sim_message = Message(content=sim_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD) new_messages = memory_storage.search(sim_message) assert len(new_messages) == 0 # similar, return [] new_conent = "Incorporate basic features of a snake game such as scoring and increasing difficulty" - new_message = Message( - content=new_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() - ) + new_message = Message(content=new_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index f0a18da46..e8350b6e6 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -26,7 +26,7 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) + role.put_message(Message(content=task, cause_by=BossRequirement)) await role._observe() await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 7623aee95..0935dd98c 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -29,7 +29,7 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.put_message(Message(content=task, cause_by=BossRequirement.get_class_name())) + role.put_message(Message(content=task, cause_by=BossRequirement)) await role._observe() await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index e67d64abc..1bf20e9b7 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -254,7 +254,7 @@ class KnowledgeBase { class MockMessages: - req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement.get_class_name()) - prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD.get_class_name()) - system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign.get_class_name()) - tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks.get_class_name()) + req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement) + prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD) + system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign) + tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks) diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index 714618852..472d4cd9d 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -51,7 +51,7 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) env.set_manager(Manager()) - env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement.get_class_name())) + env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) await env.run(k=2) logger.info(f"{env.history=}") diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index e4aa0c0dd..e18ebbe79 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -13,6 +13,7 @@ from metagpt.actions import Action from metagpt.schema import AIMessage, Message, Routes, SystemMessage, UserMessage +from metagpt.utils.common import get_class_name @pytest.mark.asyncio @@ -54,9 +55,9 @@ def test_message(): m.cause_by = "Message" assert m.cause_by == "Message" m.cause_by = Action - assert m.cause_by == Action.get_class_name() + assert m.cause_by == get_class_name(Action) m.cause_by = Action() - assert m.cause_by == Action.get_class_name() + assert m.cause_by == get_class_name(Action) m.content = "b" assert m.content == "b" diff --git a/tests/metagpt/utils/test_named.py b/tests/metagpt/utils/test_named.py deleted file mode 100644 index ff1f07205..000000000 --- a/tests/metagpt/utils/test_named.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023-11-1 -@Author : mashenquan -@File : test_named.py -""" -import pytest - -from metagpt.utils.named import Named - - -@pytest.mark.asyncio -async def test_suite(): - class A(Named): - pass - - class B(A): - pass - - assert A.get_class_name() == "tests.metagpt.utils.test_named.A" - assert A().get_object_name() == "tests.metagpt.utils.test_named.A" - assert B.get_class_name() == "tests.metagpt.utils.test_named.B" - assert B().get_object_name() == "tests.metagpt.utils.test_named.B" - - -if __name__ == "__main__": - pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_serialize.py b/tests/metagpt/utils/test_serialize.py index 7889f96fe..3f566d64d 100644 --- a/tests/metagpt/utils/test_serialize.py +++ b/tests/metagpt/utils/test_serialize.py @@ -59,7 +59,7 @@ def test_serialize_and_deserialize_message(): ic_obj = ActionOutput.create_model_class("prd", out_mapping) message = Message( - content="prd demand", instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD.get_class_name() + content="prd demand", instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD ) # WritePRD as test action message_ser = serialize_message(message) From 56f544a675ef7417b46e2f683609b497af31feef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 16:49:13 +0800 Subject: [PATCH 042/232] refactor: @cause_by.setter --- examples/agent_creator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 5a1398456..3618c0608 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -2,8 +2,6 @@ Filename: MetaGPT/examples/agent_creator.py Created Date: Tuesday, September 12th 2023, 3:28:37 pm Author: garylin2099 -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import re @@ -12,7 +10,6 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import get_object_name with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator @@ -75,7 +72,7 @@ async def _act(self) -> Message: instruction = msg.content code_text = await CreateAgent().run(example=self.agent_template, instruction=instruction) - msg = Message(content=code_text, role=self.profile, cause_by=get_object_name(todo)) + msg = Message(content=code_text, role=self.profile, cause_by=todo) return msg From 8ea52d8a83ce9615b56831fdd9c27c82e0c885f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 16:52:21 +0800 Subject: [PATCH 043/232] refactor: @cause_by.setter --- examples/build_customized_agent.py | 4 +--- metagpt/schema.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index af15c90ca..f7f554e53 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -2,8 +2,6 @@ Filename: MetaGPT/examples/build_customized_agent.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import asyncio import re @@ -83,7 +81,7 @@ async def _act(self) -> Message: instruction = msg.content code_text = await SimpleWriteCode().run(instruction) - msg = Message(content=code_text, role=self.profile, cause_by=get_object_name(todo)) + msg = Message(content=code_text, role=self.profile, cause_by=todo) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 1082c5ddb..0be067cfe 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -135,7 +135,7 @@ def __init__(self, content, **kwargs): self.set_cause_by(v) continue if k == MESSAGE_ROUTE_TO: - self.add_to(v) + self.add_to(any_to_str(v)) continue self.meta_info[k] = v From 87882bf7ab56c3bcff54b5bb8fd41c09afd5bcb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 16:54:36 +0800 Subject: [PATCH 044/232] refactor: @cause_by.setter --- examples/build_customized_agent.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index f7f554e53..ef274be8b 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -13,7 +13,6 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import get_object_name class SimpleWriteCode(Action): @@ -119,7 +118,7 @@ async def _act(self) -> Message: code_text = msg.content result = await SimpleRunCode().run(code_text) - msg = Message(content=result, role=self.profile, cause_by=get_object_name(todo)) + msg = Message(content=result, role=self.profile, cause_by=todo) self._rc.memory.add(msg) return msg From b0d451d4d60246d36ff77f28830d6ce16f846c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 16:55:51 +0800 Subject: [PATCH 045/232] refactor: @cause_by.setter --- examples/sk_agent.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/sk_agent.py b/examples/sk_agent.py index 21714cca1..a7513e838 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -4,8 +4,6 @@ @Time : 2023/9/13 12:36 @Author : femto Zheng @File : sk_agent.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import asyncio From 2129c904ea498ef54b233b94235ae5faacb6eb9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:02:40 +0800 Subject: [PATCH 046/232] refactor: @cause_by.setter --- metagpt/actions/action.py | 1 - metagpt/software_company.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index fd114b332..790295d55 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : action.py -@Modified By: mashenquan, 2023-11-1. Add generic class-to-string and object-to-string conversion functionality. """ import re from abc import ABC diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 57bd5db19..354773444 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -5,9 +5,7 @@ @Author : alexanderwu @File : software_company.py @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: - 1. Change the data type of the `cause_by` value in the `Message` to a string to support the new message - distribution feature. - 2. Abandon the design of having `Environment` store all messages. + 1. Abandon the design of having `Environment` store all messages. """ from pydantic import BaseModel, Field From 8f85d80b181825dd2d43e4c6fe24ab0c306a3e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:05:15 +0800 Subject: [PATCH 047/232] refactor: @cause_by.setter --- tests/metagpt/actions/test_write_prd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 5a121adce..07d701cb9 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -4,8 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_prd.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. +@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, replace `handle` with `run`. """ import pytest From 2b2f29dcd579675ae3f0cb30217625787918474c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:06:21 +0800 Subject: [PATCH 048/232] refactor: @cause_by.setter --- tests/metagpt/memory/test_longterm_memory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index b33dd312d..c5b5c6eb1 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- """ @Desc : unittest of `metagpt/memory/longterm_memory.py` -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from metagpt.actions import BossRequirement From 9de646c01d5b4ffb977d44e335232c840a6bce7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:07:24 +0800 Subject: [PATCH 049/232] refactor: @cause_by.setter --- tests/metagpt/memory/test_memory_storage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index c40bbbba5..251c70b02 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- """ @Desc : the unittests of metagpt/memory/memory_storage.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ From e696442db935609c842f9d855a4926b048551414 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:08:58 +0800 Subject: [PATCH 050/232] refactor: @cause_by.setter --- tests/metagpt/planner/test_action_planner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index e8350b6e6..b8d4c1ad9 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -4,9 +4,8 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py -@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data - type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution - feature for message handling. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill From e86d8a3952ec3dd28a46ff4c8a118d08ecb7249c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:09:48 +0800 Subject: [PATCH 051/232] refactor: @cause_by.setter --- tests/metagpt/planner/test_basic_planner.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 0935dd98c..24250a0b0 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -4,9 +4,8 @@ @Time : 2023/9/16 20:03 @Author : femto Zheng @File : test_basic_planner.py -@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data - type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution - feature for message handling. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest from semantic_kernel.core_skills import TextSkill From be77a9c30866cefe99105d2975d5236c67284875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:10:44 +0800 Subject: [PATCH 052/232] refactor: @cause_by.setter --- tests/metagpt/roles/mock.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 1bf20e9b7..1b02fbaa5 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -4,8 +4,6 @@ @Time : 2023/5/12 13:05 @Author : alexanderwu @File : mock.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message From 3a5bfcafc52613b5691aaa7121634d60a834f402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:12:02 +0800 Subject: [PATCH 053/232] refactor: @cause_by.setter --- tests/metagpt/roles/test_architect.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/metagpt/roles/test_architect.py b/tests/metagpt/roles/test_architect.py index 4effadaaa..111438b0b 100644 --- a/tests/metagpt/roles/test_architect.py +++ b/tests/metagpt/roles/test_architect.py @@ -4,9 +4,8 @@ @Time : 2023/5/20 14:37 @Author : alexanderwu @File : test_architect.py -@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data - type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution - feature for message handling. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest From 327c047fa51226f36a8dd414ca77c7fcde319493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:13:10 +0800 Subject: [PATCH 054/232] refactor: @cause_by.setter --- tests/metagpt/roles/test_engineer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index 93f2efb77..3dc599770 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -4,9 +4,8 @@ @Time : 2023/5/12 10:14 @Author : alexanderwu @File : test_engineer.py -@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data - type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution - feature for message handling. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message handling. """ import pytest From eba7f868e71678338125221c85f0aa2d527c16b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:17:02 +0800 Subject: [PATCH 055/232] refactor: @cause_by.setter --- tests/metagpt/test_environment.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index 472d4cd9d..a0f1f6257 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -4,8 +4,6 @@ @Time : 2023/5/12 00:47 @Author : alexanderwu @File : test_environment.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ import pytest From ed7eb4d08a07a0c4dc2530e7e1c55d6c5bae0bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 4 Nov 2023 17:18:26 +0800 Subject: [PATCH 056/232] refactor: @cause_by.setter --- tests/metagpt/utils/test_serialize.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/metagpt/utils/test_serialize.py b/tests/metagpt/utils/test_serialize.py index 3f566d64d..ffa34866c 100644 --- a/tests/metagpt/utils/test_serialize.py +++ b/tests/metagpt/utils/test_serialize.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- """ @Desc : the unittest of serialize -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from typing import List, Tuple From c6f97f748717c030f752ebe492342aace4a4ab13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 6 Nov 2023 11:47:29 +0800 Subject: [PATCH 057/232] refactor: tx_from/tx_to --- examples/debate.py | 8 ++++---- metagpt/const.py | 4 ++-- metagpt/memory/memory.py | 2 +- metagpt/roles/engineer.py | 4 ++-- metagpt/roles/qa_engineer.py | 14 +++++++------- metagpt/roles/role.py | 4 ++-- metagpt/schema.py | 16 ++++++++-------- metagpt/software_company.py | 2 +- tests/metagpt/test_role.py | 2 +- tests/metagpt/test_schema.py | 8 ++++---- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index c1d997678..77a2ce129 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -78,8 +78,8 @@ async def _act(self) -> Message: content=rsp, role=self.profile, cause_by=ShoutOut, - tx_from=self.name, - tx_to=self.opponent_name, + msg_from=self.name, + msg_to=self.opponent_name, ) return msg @@ -121,8 +121,8 @@ async def _act(self) -> Message: content=rsp, role=self.profile, cause_by=ShoutOut, - tx_from=self.name, - tx_to=self.opponent_name, + msg_from=self.name, + msg_to=self.opponent_name, ) return msg diff --git a/metagpt/const.py b/metagpt/const.py index e783ec8d0..7b8203bce 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -44,7 +44,7 @@ def get_project_root(): MEM_TTL = 24 * 30 * 3600 -MESSAGE_ROUTE_FROM = "tx_from" -MESSAGE_ROUTE_TO = "tx_to" +MESSAGE_ROUTE_FROM = "msg_from" +MESSAGE_ROUTE_TO = "msg_to" MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index cf3140bdb..c6b732076 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -28,7 +28,7 @@ def add(self, message: Message): self.storage.append(message) # According to the design of RFC 116, it allows message filtering based on different labels, thus # necessitating the creation of separate indices for each label. - for k in message.tx_to: + for k in message.msg_to: self.index[k].append(message) def add_batch(self, messages: Iterable[Message]): diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 7f05c52c5..8778471cc 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -170,7 +170,7 @@ async def _act_sp(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - tx_to="QaEngineer", + msg_to="QaEngineer", ) return msg @@ -213,7 +213,7 @@ async def _act_sp_precision(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - tx_to="QaEngineer", + msg_to="QaEngineer", ) return msg diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 64d7f9702..05fc5b217 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -100,8 +100,8 @@ async def _write_test(self, message: Message) -> None: content=str(file_info), role=self.profile, cause_by=WriteTest, - tx_from=self.profile, - tx_to=self.profile, + msg_from=self.profile, + msg_to=self.profile, ) self.publish_message(msg) @@ -133,7 +133,7 @@ async def _run_code(self, msg): recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message(content=content, role=self.profile, cause_by=RunCode, tx_from=self.profile, tx_to=recipient) + msg = Message(content=content, role=self.profile, cause_by=RunCode, msg_from=self.profile, msg_to=recipient) self.publish_message(msg) async def _debug_error(self, msg): @@ -146,8 +146,8 @@ async def _debug_error(self, msg): content=file_info, role=self.profile, cause_by=DebugError, - tx_from=self.profile, - tx_to=recipient, + msg_from=self.profile, + msg_to=recipient, ) self.publish_message(msg) @@ -164,7 +164,7 @@ async def _act(self) -> Message: content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", role=self.profile, cause_by=WriteTest, - tx_from=self.profile, + msg_from=self.profile, ) return result_msg @@ -188,6 +188,6 @@ async def _act(self) -> Message: content=f"Round {self.test_round} of tests done", role=self.profile, cause_by=WriteTest, - tx_from=self.profile, + msg_from=self.profile, ) return result_msg diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 87a03b391..9bbba2070 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -211,14 +211,14 @@ async def _act(self) -> Message: instruct_content=response.instruct_content, role=self.profile, cause_by=get_object_name(self._rc.todo), - tx_from=get_object_name(self), + msg_from=get_object_name(self), ) else: msg = Message( content=response, role=self.profile, cause_by=get_object_name(self._rc.todo), - tx_from=get_object_name(self), + msg_from=get_object_name(self), ) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 0be067cfe..39a62e706 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -77,13 +77,13 @@ def is_recipient(self, tags: Set) -> bool: return False @property - def tx_from(self): + def msg_from(self): """Message route info tells who sent this message.""" route = self._get_route() return route.get(MESSAGE_ROUTE_FROM) @property - def tx_to(self): + def msg_to(self): """Labels for the consumer to filter its subscribed messages.""" route = self._get_route() return route.get(MESSAGE_ROUTE_TO) @@ -112,8 +112,8 @@ def __init__(self, content, **kwargs): :param instruct_content: Message content struct. :param meta_info: Message meta info. :param route: Message route configuration. - :param tx_from: Message route info tells who sent this message. - :param tx_to: Labels for the consumer to filter its subscribed messages. + :param msg_from: Message route info tells who sent this message. + :param msg_to: Labels for the consumer to filter its subscribed messages. :param cause_by: Labels for the consumer to filter its subscribed messages, also serving as meta info. :param role: Message meta info tells who sent this message. """ @@ -174,14 +174,14 @@ def set_cause_by(self, val): self.route.replace(old_value, new_value) @property - def tx_from(self): + def msg_from(self): """Message route info tells who sent this message.""" - return self.route.tx_from + return self.route.msg_from @property - def tx_to(self): + def msg_to(self): """Labels for the consumer to filter its subscribed messages.""" - return self.route.tx_to + return self.route.msg_to def set_role(self, v): """Set the message's meta info indicating the sender.""" diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 354773444..1b6936870 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -53,7 +53,7 @@ def start_project(self, idea): role="BOSS", content=idea, cause_by=BossRequirement, - tx_from=SoftwareCompany, + msg_from=SoftwareCompany, ) ) diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 7794c9b57..69386e28c 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -69,7 +69,7 @@ class Input(BaseModel): env = Environment() env.add_role(role) assert env.get_subscribed_tags(role) == {seed.subscription} - env.publish_message(Message(content="test", tx_to=seed.subscription)) + env.publish_message(Message(content="test", msg_to=seed.subscription)) assert not role.is_idle while not env.is_idle: await env.run() diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index e18ebbe79..5ebc7ce1d 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -66,13 +66,13 @@ def test_message(): def test_routes(): route = Routes() route.set_from("a") - assert route.tx_from == "a" + assert route.msg_from == "a" route.add_to("b") - assert route.tx_to == {"b"} + assert route.msg_to == {"b"} route.add_to("c") - assert route.tx_to == {"b", "c"} + assert route.msg_to == {"b", "c"} route.set_to({"e", "f"}) - assert route.tx_to == {"e", "f"} + assert route.msg_to == {"e", "f"} assert route.is_recipient({"e"}) assert route.is_recipient({"f"}) assert not route.is_recipient({"a"}) From c496b6b5f604cfa46e239b360bf6a2a743114536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 6 Nov 2023 22:38:43 +0800 Subject: [PATCH 058/232] feat: add default subscriptions to all Role --- metagpt/const.py | 1 + metagpt/roles/role.py | 4 ++++ metagpt/schema.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/metagpt/const.py b/metagpt/const.py index 7b8203bce..2ba875543 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -48,3 +48,4 @@ def get_project_root(): MESSAGE_ROUTE_TO = "msg_to" MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" +MESSAGE_ROUTE_TO_ALL = "" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 9bbba2070..6e8c5e421 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -135,6 +135,10 @@ def _init_actions(self, actions): def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" tags = {get_class_name(t) for t in actions} + # Add default subscription tags for developers' direct use. + if self.name: + tags.add(self.name) + tags.add(get_object_name(self)) self.subscribe(tags) def subscribe(self, tags: Set[str]): diff --git a/metagpt/schema.py b/metagpt/schema.py index 39a62e706..fb8885614 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -22,6 +22,7 @@ MESSAGE_ROUTE_CAUSE_BY, MESSAGE_ROUTE_FROM, MESSAGE_ROUTE_TO, + MESSAGE_ROUTE_TO_ALL, ) from metagpt.logs import logger from metagpt.utils.common import any_to_str @@ -71,6 +72,8 @@ def is_recipient(self, tags: Set) -> bool: if not to_tags: return True + if MESSAGE_ROUTE_TO_ALL in to_tags: + return True for k in tags: if k in to_tags: return True From a045f73fec38536441a53f27f948ec5b9f1a5594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 6 Nov 2023 23:13:58 +0800 Subject: [PATCH 059/232] feat: Support more versatile parameter formats. --- metagpt/schema.py | 6 +++++- tests/metagpt/test_role.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index fb8885614..e89ac00ea 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -138,7 +138,11 @@ def __init__(self, content, **kwargs): self.set_cause_by(v) continue if k == MESSAGE_ROUTE_TO: - self.add_to(any_to_str(v)) + if isinstance(v, tuple) or isinstance(v, list) or isinstance(v, set): + for i in v: + self.add_to(any_to_str(i)) + else: + self.add_to(any_to_str(v)) continue self.meta_info[k] = v diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 69386e28c..447de7ee5 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -18,6 +18,7 @@ from metagpt.environment import Environment from metagpt.roles import Role from metagpt.schema import Message +from metagpt.utils.common import get_class_name class MockAction(Action): @@ -84,5 +85,17 @@ class Input(BaseModel): assert env.get_subscribed_tags(role) == {seed.subscription, tag} +@pytest.mark.asyncio +async def test_msg_to(): + m = Message(content="a", msg_to=["a", MockRole, Message]) + assert m.msg_to == {"a", get_class_name(MockRole), get_class_name(Message)} + + m = Message(content="a", cause_by=MockAction, msg_to={"a", MockRole, Message}) + assert m.msg_to == {"a", get_class_name(MockRole), get_class_name(Message), get_class_name(MockAction)} + + m = Message(content="a", msg_to=("a", MockRole, Message)) + assert m.msg_to == {"a", get_class_name(MockRole), get_class_name(Message)} + + if __name__ == "__main__": pytest.main([__file__, "-s"]) From 93ebe8c103388ffbe48119b0600ea0bd4c55b64b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 7 Nov 2023 14:12:20 +0800 Subject: [PATCH 060/232] feat: recover `history` --- metagpt/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/environment.py b/metagpt/environment.py index a7e6322ff..75a790714 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -29,6 +29,7 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) consumers: dict[Role, Set] = Field(default_factory=dict) + history: str = Field(default="") # For debug class Config: arbitrary_types_allowed = True @@ -67,6 +68,7 @@ def publish_message(self, message: Message) -> bool: found = True if not found: logger.warning(f"Message no recipients: {message.dump()}") + self.history += f"\n{message}" # For debug return True From af4c87e1234db2828e3f76a7db17b1ceb7ba81ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 13:42:08 +0800 Subject: [PATCH 061/232] refactor: rename is_recipient --- examples/debate.py | 4 ++-- metagpt/actions/write_code.py | 2 +- metagpt/environment.py | 2 +- metagpt/memory/longterm_memory.py | 4 ++-- metagpt/roles/engineer.py | 2 +- metagpt/roles/qa_engineer.py | 8 ++++---- metagpt/schema.py | 8 ++++---- tests/metagpt/test_schema.py | 8 ++++---- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index 77a2ce129..cf0c0124c 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -60,7 +60,7 @@ def __init__( async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.is_recipient({self.name})] + self._rc.news = [msg for msg in self._rc.news if msg.contain_any({self.name})] return len(self._rc.news) async def _act(self) -> Message: @@ -103,7 +103,7 @@ async def _observe(self) -> int: # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, # disregard own messages from the last round message_filter = {BossRequirement, self.name} - self._rc.news = [msg for msg in self._rc.news if msg.is_recipient(message_filter)] + self._rc.news = [msg for msg in self._rc.news if msg.contain_any(message_filter)] return len(self._rc.news) async def _act(self) -> Message: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 8b6451134..f2a4744d9 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -59,7 +59,7 @@ def _save(self, context, filename, code): return message_filter = {WriteDesign} - design = [i for i in context if i.is_recipient(message_filter)][0] + design = [i for i in context if i.contain_any(message_filter)][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) ws_path = WORKSPACE_ROOT / ws_name diff --git a/metagpt/environment.py b/metagpt/environment.py index 75a790714..fb564e1ab 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -63,7 +63,7 @@ def publish_message(self, message: Message) -> bool: found = False # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 for obj, subscribed_tags in self.consumers.items(): - if message.is_recipient(subscribed_tags): + if message.contain_any(subscribed_tags): obj.put_message(message) found = True if not found: diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index e73ae334e..2a4b604e0 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -3,7 +3,7 @@ """ @Desc : the implement of Long-term memory @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: - 1. Replace code related to message filtering with the `Message.is_recipient` function. + 1. Replace code related to message filtering with the `Message.contain_any` function. """ from metagpt.logs import logger @@ -40,7 +40,7 @@ def recover_memory(self, role_id: str, rc: "RoleContext"): def add(self, message: Message): super(LongTermMemory, self).add(message) - if message.is_recipient(self.rc.watch) and not self.msg_from_recover: + if message.contain_any(self.rc.watch) and not self.msg_from_recover: # currently, only add role's watching messages to its memory_storage # and ignore adding messages from recover repeatedly self.memory_storage.add(message) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 8778471cc..882cf89dd 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -233,7 +233,7 @@ async def _observe(self) -> int: # Parse task lists message_filter = {WriteTasks} for message in self._rc.news: - if not message.is_recipient(message_filter): + if not message.contain_any(message_filter): continue self.todos = self.parse_tasks(message) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 05fc5b217..104aa3dfb 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -154,7 +154,7 @@ async def _debug_error(self, msg): async def _observe(self) -> int: await super()._observe() self._rc.news = [ - msg for msg in self._rc.news if msg.is_recipient({self.profile}) + msg for msg in self._rc.news if msg.contain_any({self.profile}) ] # only relevant msgs count as observed news return len(self._rc.news) @@ -174,13 +174,13 @@ async def _act(self) -> Message: for msg in self._rc.news: # Decide what to do based on observed msg type, currently defined by human, # might potentially be moved to _think, that is, let the agent decides for itself - if msg.is_recipient(code_filters): + if msg.contain_any(code_filters): # engineer wrote a code, time to write a test for it await self._write_test(msg) - elif msg.is_recipient(test_filters): + elif msg.contain_any(test_filters): # I wrote or debugged my test code, time to run it await self._run_code(msg) - elif msg.is_recipient(run_filters): + elif msg.contain_any(run_filters): # I ran my test code, time to fix bugs, if any await self._debug_error(msg) self.test_round += 1 diff --git a/metagpt/schema.py b/metagpt/schema.py index e89ac00ea..1b00843a6 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -65,8 +65,8 @@ def _get_route(self) -> Dict: self.routes.append({}) return self.routes[0] - def is_recipient(self, tags: Set) -> bool: - """Check if it is the message recipient.""" + def contain_any(self, tags: Set) -> bool: + """Check if this object contains these tags.""" route = self._get_route() to_tags = route.get(MESSAGE_ROUTE_TO) if not to_tags: @@ -206,9 +206,9 @@ def add_to(self, tag: str): """Add a subscription label for the recipients.""" self.route.add_to(tag) - def is_recipient(self, tags: Set): + def contain_any(self, tags: Set): """Return true if any input label exists in the message's subscription labels.""" - return self.route.is_recipient(tags) + return self.route.contain_any(tags) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 5ebc7ce1d..05127362b 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -48,7 +48,7 @@ def test_message(): m = Message("a", role="b", cause_by="c", x="d") assert m.content == "a" assert m.role == "b" - assert m.is_recipient({"c"}) + assert m.contain_any({"c"}) assert m.cause_by == "c" assert m.get_meta("x") == "d" @@ -73,9 +73,9 @@ def test_routes(): assert route.msg_to == {"b", "c"} route.set_to({"e", "f"}) assert route.msg_to == {"e", "f"} - assert route.is_recipient({"e"}) - assert route.is_recipient({"f"}) - assert not route.is_recipient({"a"}) + assert route.contain_any({"e"}) + assert route.contain_any({"f"}) + assert not route.contain_any({"a"}) if __name__ == "__main__": From c18bc7c876f062c3427159146d8274d7012979d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 20:27:18 +0800 Subject: [PATCH 062/232] refactor: Simplify the Message class. --- metagpt/const.py | 4 +- metagpt/schema.py | 237 ++++++----------------------------- metagpt/utils/common.py | 11 ++ tests/metagpt/test_schema.py | 28 ++--- 4 files changed, 64 insertions(+), 216 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 2ba875543..fa0ccc536 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -44,8 +44,8 @@ def get_project_root(): MEM_TTL = 24 * 30 * 3600 -MESSAGE_ROUTE_FROM = "msg_from" -MESSAGE_ROUTE_TO = "msg_to" +MESSAGE_ROUTE_FROM = "sent_from" +MESSAGE_ROUTE_TO = "send_to" MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" MESSAGE_ROUTE_TO_ALL = "" diff --git a/metagpt/schema.py b/metagpt/schema.py index 1b00843a6..7fdcef2ed 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -13,19 +13,18 @@ import json from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError -from typing import Dict, List, Set, TypedDict +from typing import List, Set, TypedDict from pydantic import BaseModel, Field from metagpt.const import ( - MESSAGE_META_ROLE, MESSAGE_ROUTE_CAUSE_BY, MESSAGE_ROUTE_FROM, MESSAGE_ROUTE_TO, MESSAGE_ROUTE_TO_ALL, ) from metagpt.logs import logger -from metagpt.utils.common import any_to_str +from metagpt.utils.common import any_to_str, any_to_str_set class RawMessage(TypedDict): @@ -33,182 +32,56 @@ class RawMessage(TypedDict): role: str -class Routes(BaseModel): - """Responsible for managing routing information for the Message class.""" - - routes: List[Dict] = Field(default_factory=list) - - def set_from(self, value): - """Set the label of the message sender.""" - route = self._get_route() - route[MESSAGE_ROUTE_FROM] = value - - def set_to(self, tags: Set): - """Set the labels of the message recipient.""" - route = self._get_route() - if tags: - route[MESSAGE_ROUTE_TO] = tags - return - - if MESSAGE_ROUTE_TO in route: - del route[MESSAGE_ROUTE_TO] - - def add_to(self, tag: str): - """Add a label of the message recipient.""" - route = self._get_route() - tags = route.get(MESSAGE_ROUTE_TO, set()) - tags.add(tag) - route[MESSAGE_ROUTE_TO] = tags - - def _get_route(self) -> Dict: - if not self.routes: - self.routes.append({}) - return self.routes[0] - - def contain_any(self, tags: Set) -> bool: - """Check if this object contains these tags.""" - route = self._get_route() - to_tags = route.get(MESSAGE_ROUTE_TO) - if not to_tags: - return True - - if MESSAGE_ROUTE_TO_ALL in to_tags: - return True - for k in tags: - if k in to_tags: - return True - return False - - @property - def msg_from(self): - """Message route info tells who sent this message.""" - route = self._get_route() - return route.get(MESSAGE_ROUTE_FROM) - - @property - def msg_to(self): - """Labels for the consumer to filter its subscribed messages.""" - route = self._get_route() - return route.get(MESSAGE_ROUTE_TO) - - def replace(self, old_val, new_val): - """Replace old value with new value""" - route = self._get_route() - tags = route.get(MESSAGE_ROUTE_TO, set()) - tags.discard(old_val) - tags.add(new_val) - route[MESSAGE_ROUTE_TO] = tags - - class Message(BaseModel): """list[: ]""" content: str - instruct_content: BaseModel = None - meta_info: Dict = Field(default_factory=dict) - route: Routes = Field(default_factory=Routes) - - def __init__(self, content, **kwargs): + instruct_content: BaseModel = Field(default=None) + role: str = "user" # system / user / assistant + cause_by: str = "" + sent_from: str = "" + send_to: Set = Field(default_factory=set) + + def __init__( + self, + content, + instruct_content=None, + role="user", + cause_by="", + sent_from="", + send_to=MESSAGE_ROUTE_TO_ALL, + **kwargs, + ): """ Parameters not listed below will be stored as meta info, including custom parameters. :param content: Message content. :param instruct_content: Message content struct. - :param meta_info: Message meta info. - :param route: Message route configuration. - :param msg_from: Message route info tells who sent this message. - :param msg_to: Labels for the consumer to filter its subscribed messages. - :param cause_by: Labels for the consumer to filter its subscribed messages, also serving as meta info. + :param cause_by: Message producer + :param sent_from: Message route info tells who sent this message. + :param send_to: Labels for the consumer to filter its subscribed messages. :param role: Message meta info tells who sent this message. """ - super(Message, self).__init__( - content=content or kwargs.get("content"), - instruct_content=kwargs.get("instruct_content"), - meta_info=kwargs.get("meta_info", {}), - route=kwargs.get("route", Routes()), + super().__init__( + content=content, + instruct_content=instruct_content, + role=role, + cause_by=any_to_str(cause_by), + sent_from=any_to_str(sent_from), + send_to=any_to_str_set(send_to), + **kwargs, ) - attribute_names = Message.__annotations__.keys() - for k, v in kwargs.items(): - if k in attribute_names: - continue - if k == MESSAGE_ROUTE_FROM: - self.set_from(any_to_str(v)) - continue - if k == MESSAGE_ROUTE_CAUSE_BY: - self.set_cause_by(v) - continue - if k == MESSAGE_ROUTE_TO: - if isinstance(v, tuple) or isinstance(v, list) or isinstance(v, set): - for i in v: - self.add_to(any_to_str(i)) - else: - self.add_to(any_to_str(v)) - continue - self.meta_info[k] = v - - def get_meta(self, key): - """Get meta info""" - return self.meta_info.get(key) - - def set_meta(self, key, value): - """Set meta info""" - self.meta_info[key] = value - - @property - def role(self): - """Message meta info tells who sent this message.""" - return self.get_meta(MESSAGE_META_ROLE) - - @property - def cause_by(self): - """Labels for the consumer to filter its subscribed messages, also serving as meta info.""" - return self.get_meta(MESSAGE_ROUTE_CAUSE_BY) - def __setattr__(self, key, val): - """Override `@property.setter`""" + """Override `@property.setter`, convert non-string parameters into string parameters.""" if key == MESSAGE_ROUTE_CAUSE_BY: - self.set_cause_by(val) - return - if key == MESSAGE_ROUTE_FROM: - self.set_from(any_to_str(val)) - super().__setattr__(key, val) - - def set_cause_by(self, val): - """Update the value of `cause_by` in the `meta_info` and `routes` attributes.""" - old_value = self.get_meta(MESSAGE_ROUTE_CAUSE_BY) - new_value = any_to_str(val) - self.set_meta(MESSAGE_ROUTE_CAUSE_BY, new_value) - self.route.replace(old_value, new_value) - - @property - def msg_from(self): - """Message route info tells who sent this message.""" - return self.route.msg_from - - @property - def msg_to(self): - """Labels for the consumer to filter its subscribed messages.""" - return self.route.msg_to - - def set_role(self, v): - """Set the message's meta info indicating the sender.""" - self.set_meta(MESSAGE_META_ROLE, v) - - def set_from(self, v): - """Set the message's meta info indicating the sender.""" - self.route.set_from(v) - - def set_to(self, tags: Set): - """Set the message's meta info indicating the sender.""" - self.route.set_to(tags) - - def add_to(self, tag: str): - """Add a subscription label for the recipients.""" - self.route.add_to(tag) - - def contain_any(self, tags: Set): - """Return true if any input label exists in the message's subscription labels.""" - return self.route.contain_any(tags) + new_val = any_to_str(val) + elif key == MESSAGE_ROUTE_FROM: + new_val = any_to_str(val) + elif key == MESSAGE_ROUTE_TO: + new_val = any_to_str_set(val) + else: + new_val = val + super().__setattr__(key, new_val) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) @@ -226,13 +99,13 @@ def dump(self) -> str: return self.json(exclude_none=True) @staticmethod - def load(v): + def load(val): """Convert the json string to object.""" try: - d = json.loads(v) + d = json.loads(val) return Message(**d) except JSONDecodeError as err: - logger.error(f"parse json failed: {v}, error:{err}") + logger.error(f"parse json failed: {val}, error:{err}") return None @@ -327,31 +200,3 @@ def load(self, v) -> "MessageQueue": logger.warning(f"JSON load failed: {v}, error:{e}") return q - - -if __name__ == "__main__": - m = Message("a", role="v1") - m.set_role("v2") - v = m.dump() - m = Message.load(v) - m.cause_by = "Message" - m.cause_by = Routes - m.cause_by = Routes() - m.content = "b" - - test_content = "test_message" - msgs = [ - UserMessage(test_content), - SystemMessage(test_content), - AIMessage(test_content), - Message(test_content, role="QA"), - ] - logger.info(msgs) - - jsons = [ - UserMessage(test_content).dump(), - SystemMessage(test_content).dump(), - AIMessage(test_content).dump(), - Message(test_content, role="QA").dump(), - ] - logger.info(jsons) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index b372f0d8d..cd42b1412 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -325,3 +325,14 @@ def any_to_str(val) -> str: return get_object_name(val) return get_class_name(val) + + +def any_to_str_set(val) -> set: + """Convert any type to string set.""" + res = set() + if isinstance(val, dict) or isinstance(val, list) or isinstance(val, set) or isinstance(val, tuple): + for i in val: + res.add(any_to_str(i)) + else: + res.add(any_to_str(val)) + return res diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 05127362b..51ebd5baa 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -12,7 +12,7 @@ import pytest from metagpt.actions import Action -from metagpt.schema import AIMessage, Message, Routes, SystemMessage, UserMessage +from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage from metagpt.utils.common import get_class_name @@ -37,20 +37,19 @@ def test_message(): d = json.loads(v) assert d assert d.get("content") == "a" - assert d.get("meta_info") == {"role": "v1"} - m.set_role("v2") + assert d.get("role") == "v1" + m.role = "v2" v = m.dump() assert v m = Message.load(v) assert m.content == "a" assert m.role == "v2" - m = Message("a", role="b", cause_by="c", x="d") + m = Message("a", role="b", cause_by="c", x="d", send_to="c") assert m.content == "a" assert m.role == "b" - assert m.contain_any({"c"}) + assert m.send_to == {"c"} assert m.cause_by == "c" - assert m.get_meta("x") == "d" m.cause_by = "Message" assert m.cause_by == "Message" @@ -64,18 +63,11 @@ def test_message(): @pytest.mark.asyncio def test_routes(): - route = Routes() - route.set_from("a") - assert route.msg_from == "a" - route.add_to("b") - assert route.msg_to == {"b"} - route.add_to("c") - assert route.msg_to == {"b", "c"} - route.set_to({"e", "f"}) - assert route.msg_to == {"e", "f"} - assert route.contain_any({"e"}) - assert route.contain_any({"f"}) - assert not route.contain_any({"a"}) + m = Message("a", role="b", cause_by="c", x="d", send_to="c") + m.send_to = "b" + assert m.send_to == {"b"} + m.send_to = {"e", Action} + assert m.send_to == {"e", get_class_name(Action)} if __name__ == "__main__": From 09fe4593f6621aa4689f9d4697711a1bc9851de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 20:36:42 +0800 Subject: [PATCH 063/232] refactor: According to RFC 116: Updated the type of index key. --- metagpt/memory/memory.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index c6b732076..7f04be63d 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -26,10 +26,8 @@ def add(self, message: Message): if message in self.storage: return self.storage.append(message) - # According to the design of RFC 116, it allows message filtering based on different labels, thus - # necessitating the creation of separate indices for each label. - for k in message.msg_to: - self.index[k].append(message) + if message.cause_by: + self.index[message.cause_by].append(message) def add_batch(self, messages: Iterable[Message]): for message in messages: From e5c792e51277677705d46716ad71bbe074e284cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 20:36:50 +0800 Subject: [PATCH 064/232] refactor: According to RFC 116: Updated the type of index key. --- metagpt/memory/memory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 7f04be63d..84289091f 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,8 +4,7 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: - Updated the message filtering logic. +@Modified By: mashenquan, 2023-11-1. According to RFC 116: Updated the type of index key. """ from collections import defaultdict from typing import Iterable, Set From 47d47d274e5d0d7e806d387db169019df7d961ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 20:44:39 +0800 Subject: [PATCH 065/232] refactor: According to RFC 113, add message dispatching functionality. --- metagpt/environment.py | 3 ++- metagpt/utils/common.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index fb564e1ab..81b5c2ac7 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -19,6 +19,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message +from metagpt.utils.common import is_subscribed class Environment(BaseModel): @@ -63,7 +64,7 @@ def publish_message(self, message: Message) -> bool: found = False # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 for obj, subscribed_tags in self.consumers.items(): - if message.contain_any(subscribed_tags): + if is_subscribed(message, subscribed_tags): obj.put_message(message) found = True if not found: diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index cd42b1412..798acf214 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -15,6 +15,7 @@ import re from typing import List, Tuple, Union +from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.logs import logger @@ -336,3 +337,14 @@ def any_to_str_set(val) -> set: else: res.add(any_to_str(val)) return res + + +def is_subscribed(message, tags): + """Return whether it's consumer""" + if MESSAGE_ROUTE_TO_ALL in message.send_to: + return True + + for t in tags: + if t in message.send_to: + return True + return False From 7e83a4bb3159279855a5eca98f97efdbf468acfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 20:56:05 +0800 Subject: [PATCH 066/232] refactor: According to RFC 116: Updated the type of send_to. --- examples/debate.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index cf0c0124c..7b03f785b 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -60,7 +60,7 @@ def __init__( async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.contain_any({self.name})] + self._rc.news = [msg for msg in self._rc.news if msg.send_to == {self.name}] return len(self._rc.news) async def _act(self) -> Message: @@ -78,8 +78,8 @@ async def _act(self) -> Message: content=rsp, role=self.profile, cause_by=ShoutOut, - msg_from=self.name, - msg_to=self.opponent_name, + sent_from=self.name, + send_to=self.opponent_name, ) return msg @@ -102,8 +102,7 @@ async def _observe(self) -> int: await super()._observe() # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, # disregard own messages from the last round - message_filter = {BossRequirement, self.name} - self._rc.news = [msg for msg in self._rc.news if msg.contain_any(message_filter)] + self._rc.news = [msg for msg in self._rc.news if msg.cause_by == BossRequirement or msg.send_to == {self.name}] return len(self._rc.news) async def _act(self) -> Message: @@ -121,8 +120,8 @@ async def _act(self) -> Message: content=rsp, role=self.profile, cause_by=ShoutOut, - msg_from=self.name, - msg_to=self.opponent_name, + sent_from=self.name, + send_to=self.opponent_name, ) return msg From 01f23d633ee2e688e7d423b448af810e6f9f1161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 21:38:52 +0800 Subject: [PATCH 067/232] refactor: In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `send_to` value of the `Message` object. --- examples/debate.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index 7b03f785b..87ac7050f 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -2,9 +2,8 @@ Filename: MetaGPT/examples/debate.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data - type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution - feature for message filtering. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `send_to` + value of the `Message` object. """ import asyncio import platform From c502b1403a6c2fb2e15f25040528873abb1eb869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 21:43:17 +0800 Subject: [PATCH 068/232] refactor: In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. --- metagpt/actions/write_code.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index f2a4744d9..be8690314 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -4,8 +4,8 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of - the `cause_by` value in the `Message` to a string to support the new message distribution feature. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` + value of the `Message` object. """ from tenacity import retry, stop_after_attempt, wait_fixed @@ -14,7 +14,7 @@ from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.common import CodeParser +from metagpt.utils.common import CodeParser, get_class_name PROMPT_TEMPLATE = """ NOTICE @@ -58,8 +58,7 @@ def _save(self, context, filename, code): if self._is_invalid(filename): return - message_filter = {WriteDesign} - design = [i for i in context if i.contain_any(message_filter)][0] + design = [i for i in context if i.cause_by == get_class_name(WriteDesign)][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) ws_path = WORKSPACE_ROOT / ws_name From d9939f437ad10f3b87959fa615ad4168f0f4e1d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 21:57:58 +0800 Subject: [PATCH 069/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/role.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 6e8c5e421..ac8a2d702 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -116,6 +116,9 @@ def __init__(self, name="", profile="", goal="", constraints="", desc=""): self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() + self._subscription = {get_object_name(self)} + if name: + self._subscription.add(name) def _reset(self): self._states = [] @@ -133,21 +136,15 @@ def _init_actions(self, actions): self._states.append(f"{idx}. {action}") def _watch(self, actions: Iterable[Type[Action]]): - """Listen to the corresponding behaviors""" + """Listen to the corresponding behaviors in private message buffer""" tags = {get_class_name(t) for t in actions} - # Add default subscription tags for developers' direct use. - if self.name: - tags.add(self.name) - tags.add(get_object_name(self)) - self.subscribe(tags) + self._rc.watch.update(tags) def subscribe(self, tags: Set[str]): """Listen to the corresponding behaviors""" - self._rc.watch.update(tags) - # check RoleContext after adding watch actions - self._rc.check(self._role_id) + self._subscription = tags if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 - self._rc.env.set_subscribed_tags(self, self.subscribed_tags) + self._rc.env.set_subscribed_tags(self, self._subscription) def _set_state(self, state): """Update the current state.""" @@ -159,6 +156,8 @@ def set_env(self, env: "Environment"): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env + if env: + env.set_subscribed_tags(self, self._subscription) @property def profile(self): From d232725a2991a2e43da9e07094d0cbc23b7a6a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 22:01:55 +0800 Subject: [PATCH 070/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/role.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index ac8a2d702..5bc241352 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -170,17 +170,9 @@ def name(self): return self._setting.name @property - def subscribed_tags(self) -> Set: + def subscription(self) -> Set: """The labels for messages to be consumed by the Role object.""" - if self._rc.watch: - return self._rc.watch - return { - self.name, - get_object_name(self), - self.profile, - f"{self.name}({self.profile})", - f"{self.name}({get_object_name(self)})", - } + return self._subscription def _get_prefix(self): """Get the role prefix""" From f47977daa81357b552cd0e791dc798826b45bf31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 22:03:06 +0800 Subject: [PATCH 071/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/role.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 5bc241352..32fa16e6a 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -139,6 +139,8 @@ def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors in private message buffer""" tags = {get_class_name(t) for t in actions} self._rc.watch.update(tags) + # check RoleContext after adding watch actions + self._rc.check(self._role_id) def subscribe(self, tags: Set[str]): """Listen to the corresponding behaviors""" From 9cdbc0a0ae93c8295756b191faa05972075e9015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 22:06:46 +0800 Subject: [PATCH 072/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/role.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 32fa16e6a..5f54e57e0 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -208,15 +208,9 @@ async def _act(self) -> Message: instruct_content=response.instruct_content, role=self.profile, cause_by=get_object_name(self._rc.todo), - msg_from=get_object_name(self), ) else: - msg = Message( - content=response, - role=self.profile, - cause_by=get_object_name(self._rc.todo), - msg_from=get_object_name(self), - ) + msg = Message(content=response, role=self.profile, cause_by=get_object_name(self._rc.todo)) return msg From d1977f15864bb2f8386766f184ddf14daf856325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 22:22:09 +0800 Subject: [PATCH 073/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 5f54e57e0..59342fa99 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -146,7 +146,7 @@ def subscribe(self, tags: Set[str]): """Listen to the corresponding behaviors""" self._subscription = tags if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 - self._rc.env.set_subscribed_tags(self, self._subscription) + self._rc.env.set_subscription(self, self._subscription) def _set_state(self, state): """Update the current state.""" From 7a2193c3d26aee1cc4c9aa9ed2c7702305770bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 22:22:44 +0800 Subject: [PATCH 074/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 59342fa99..f3e11b294 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -159,7 +159,7 @@ def set_env(self, env: "Environment"): messages by observing.""" self._rc.env = env if env: - env.set_subscribed_tags(self, self._subscription) + env.set_subscription(self, self._subscription) @property def profile(self): From 1ff99b95acaecc95c35dda8f5cbad5d0e421dc89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 22:51:12 +0800 Subject: [PATCH 075/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/environment.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index 81b5c2ac7..e9a5c6467 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -41,8 +41,6 @@ def add_role(self, role: Role): """ role.set_env(self) self.roles[role.profile] = role - # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 - self.set_subscribed_tags(role, role.subscribed_tags) def add_roles(self, roles: Iterable[Role]): """增加一批在当前环境的角色 @@ -63,8 +61,8 @@ def publish_message(self, message: Message) -> bool: logger.info(f"publish_message: {message.dump()}") found = False # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 - for obj, subscribed_tags in self.consumers.items(): - if is_subscribed(message, subscribed_tags): + for obj, subscription in self.consumers.items(): + if is_subscribed(message, subscription): obj.put_message(message) found = True if not found: @@ -106,10 +104,10 @@ def is_idle(self): return False return True - def get_subscribed_tags(self, obj): + def get_subscription(self, obj): """Get the labels for messages to be consumed by the object.""" return self.consumers.get(obj, {}) - def set_subscribed_tags(self, obj, tags): + def set_subscription(self, obj, tags): """Set the labels for message to be consumed by the object""" self.consumers[obj] = tags From c9f9c5c73e4386f4ea26dd49c65276c7e2d6eaaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 22:57:53 +0800 Subject: [PATCH 076/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/memory/longterm_memory.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 2a4b604e0..6e23a79ae 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -40,10 +40,11 @@ def recover_memory(self, role_id: str, rc: "RoleContext"): def add(self, message: Message): super(LongTermMemory, self).add(message) - if message.contain_any(self.rc.watch) and not self.msg_from_recover: - # currently, only add role's watching messages to its memory_storage - # and ignore adding messages from recover repeatedly - self.memory_storage.add(message) + for action in self.rc.watch: + if message.cause_by == action and not self.msg_from_recover: + # currently, only add role's watching messages to its memory_storage + # and ignore adding messages from recover repeatedly + self.memory_storage.add(message) def find_news(self, observed: list[Message], k=0) -> list[Message]: """ From 894a2fd593734c7e8611b0033304884dac6d9397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 23:36:56 +0800 Subject: [PATCH 077/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/memory/longterm_memory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 6e23a79ae..6fc8050ef 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- """ @Desc : the implement of Long-term memory -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: - 1. Replace code related to message filtering with the `Message.contain_any` function. """ from metagpt.logs import logger From be19d9edcb88e6aa0ca2b8b7980f20191a903359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 23:49:47 +0800 Subject: [PATCH 078/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/engineer.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 882cf89dd..03519e0ef 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -21,7 +21,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import CodeParser, get_object_name +from metagpt.utils.common import CodeParser, get_class_name, get_object_name from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -170,7 +170,7 @@ async def _act_sp(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - msg_to="QaEngineer", + send_to="QaEngineer", ) return msg @@ -185,8 +185,7 @@ async def _act_sp_precision(self) -> Message: TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ context = [] - msg_filters = [WriteDesign, WriteTasks, WriteCode] - msg = self._rc.memory.get_by_actions(msg_filters) + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) context_str = "\n".join(context) @@ -213,7 +212,7 @@ async def _act_sp_precision(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - msg_to="QaEngineer", + send_to="QaEngineer", ) return msg @@ -231,9 +230,8 @@ async def _observe(self) -> int: return ret # Parse task lists - message_filter = {WriteTasks} for message in self._rc.news: - if not message.contain_any(message_filter): + if not message.cause_by == get_class_name(WriteTasks): continue self.todos = self.parse_tasks(message) From fba70452f3c8aba12699eb2cbf7e5a62f7655ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 23:51:28 +0800 Subject: [PATCH 079/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/engineer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 03519e0ef..000e81873 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -239,7 +239,7 @@ async def _observe(self) -> int: async def _think(self) -> None: # In asynchronous scenarios, first check if the required messages are ready. - filters = {WriteTasks} + filters = {get_class_name(WriteTasks)} msgs = self._rc.memory.get_by_actions(filters) if not msgs: self._rc.todo = None From ac32cb5a67934c117108ebdde6d718b2206c51f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 8 Nov 2023 23:54:03 +0800 Subject: [PATCH 080/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/engineer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 000e81873..423fff68e 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -19,7 +19,7 @@ from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.roles import Role +from metagpt.roles import QaEngineer, Role from metagpt.schema import Message from metagpt.utils.common import CodeParser, get_class_name, get_object_name from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -170,7 +170,7 @@ async def _act_sp(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - send_to="QaEngineer", + send_to=QaEngineer, ) return msg @@ -212,7 +212,7 @@ async def _act_sp_precision(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - send_to="QaEngineer", + send_to=QaEngineer, ) return msg From df4ff5f701daf29b802381efe3a00e7080a3e447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 9 Nov 2023 00:01:51 +0800 Subject: [PATCH 081/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/roles/qa_engineer.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 104aa3dfb..760b65736 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -22,7 +22,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import CodeParser, parse_recipient +from metagpt.utils.common import CodeParser, any_to_str_set, parse_recipient from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -100,8 +100,8 @@ async def _write_test(self, message: Message) -> None: content=str(file_info), role=self.profile, cause_by=WriteTest, - msg_from=self.profile, - msg_to=self.profile, + sent_from=self.profile, + send_to=self.profile, ) self.publish_message(msg) @@ -133,7 +133,7 @@ async def _run_code(self, msg): recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message(content=content, role=self.profile, cause_by=RunCode, msg_from=self.profile, msg_to=recipient) + msg = Message(content=content, role=self.profile, cause_by=RunCode, sent_from=self.profile, send_to=recipient) self.publish_message(msg) async def _debug_error(self, msg): @@ -146,15 +146,15 @@ async def _debug_error(self, msg): content=file_info, role=self.profile, cause_by=DebugError, - msg_from=self.profile, - msg_to=recipient, + sent_from=self.profile, + send_to=recipient, ) self.publish_message(msg) async def _observe(self) -> int: await super()._observe() self._rc.news = [ - msg for msg in self._rc.news if msg.contain_any({self.profile}) + msg for msg in self._rc.news if self.profile in msg.send_to ] # only relevant msgs count as observed news return len(self._rc.news) @@ -164,23 +164,23 @@ async def _act(self) -> Message: content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", role=self.profile, cause_by=WriteTest, - msg_from=self.profile, + sent_from=self.profile, ) return result_msg - code_filters = {WriteCode, WriteCodeReview} - test_filters = {WriteTest, DebugError} - run_filters = {RunCode} + code_filters = any_to_str_set({WriteCode, WriteCodeReview}) + test_filters = any_to_str_set({WriteTest, DebugError}) + run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: # Decide what to do based on observed msg type, currently defined by human, # might potentially be moved to _think, that is, let the agent decides for itself - if msg.contain_any(code_filters): + if msg.cause_by in code_filters: # engineer wrote a code, time to write a test for it await self._write_test(msg) - elif msg.contain_any(test_filters): + elif msg.cause_by in test_filters: # I wrote or debugged my test code, time to run it await self._run_code(msg) - elif msg.contain_any(run_filters): + elif msg.cause_by in run_filters: # I ran my test code, time to fix bugs, if any await self._debug_error(msg) self.test_round += 1 @@ -188,6 +188,6 @@ async def _act(self) -> Message: content=f"Round {self.test_round} of tests done", role=self.profile, cause_by=WriteTest, - msg_from=self.profile, + sent_from=self.profile, ) return result_msg From c4ac0c72d7053a67e1c0679fa79582862328507c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 9 Nov 2023 00:41:29 +0800 Subject: [PATCH 082/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/software_company.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 1b6936870..d12998242 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -48,14 +48,7 @@ def _check_balance(self): def start_project(self, idea): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message( - Message( - role="BOSS", - content=idea, - cause_by=BossRequirement, - msg_from=SoftwareCompany, - ) - ) + self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) def _save(self): logger.info(self.json()) @@ -68,3 +61,4 @@ async def run(self, n_round=3): logger.debug(f"{n_round=}") self._check_balance() await self.environment.run() + return self.environment.history From bb050142f708557785e80ea16eff5ca3b17aed56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 9 Nov 2023 00:42:52 +0800 Subject: [PATCH 083/232] refactor: Update according to Chapter 2.1.3.2 of RFC 116 --- metagpt/software_company.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index d12998242..d3c2c463b 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,8 +4,6 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116: - 1. Abandon the design of having `Environment` store all messages. """ from pydantic import BaseModel, Field From 7504ed57570152aaa23f196ef9945fa5f552ee98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 9 Nov 2023 10:02:26 +0800 Subject: [PATCH 084/232] fixbug: recursive import --- metagpt/roles/engineer.py | 6 +++--- metagpt/schema.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 423fff68e..ba622429b 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -19,7 +19,7 @@ from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.roles import QaEngineer, Role +from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import CodeParser, get_class_name, get_object_name from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -170,7 +170,7 @@ async def _act_sp(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - send_to=QaEngineer, + send_to="Edward", ) return msg @@ -212,7 +212,7 @@ async def _act_sp_precision(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=get_object_name(self._rc.todo), - send_to=QaEngineer, + send_to="Edward", ) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 7fdcef2ed..63fe41232 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -40,7 +40,7 @@ class Message(BaseModel): role: str = "user" # system / user / assistant cause_by: str = "" sent_from: str = "" - send_to: Set = Field(default_factory=set) + send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) def __init__( self, From ea9875a7fc63b79181983bc8821cd1ab28be20dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 9 Nov 2023 11:14:05 +0800 Subject: [PATCH 085/232] fixbug: recursive import --- examples/debate.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index 87ac7050f..8f5012d66 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -3,7 +3,7 @@ Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `send_to` - value of the `Message` object. + value of the `Message` object; modify the argument type of `get_by_actions`. """ import asyncio import platform @@ -15,6 +15,7 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.software_company import SoftwareCompany +from metagpt.utils.common import any_to_str_set class ShoutOut(Action): @@ -65,7 +66,7 @@ async def _observe(self) -> int: async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions([ShoutOut]) + msg_history = self._rc.memory.get_by_actions(any_to_str_set([ShoutOut])) context = [] for m in msg_history: context.append(str(m)) @@ -107,7 +108,7 @@ async def _observe(self) -> int: async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) + msg_history = self._rc.memory.get_by_actions(any_to_str_set([BossRequirement, ShoutOut])) context = [] for m in msg_history: context.append(str(m)) From ff11cf69afa2872673a42d0910a591a3400a200a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 9 Nov 2023 11:21:20 +0800 Subject: [PATCH 086/232] fixbug: utilize the new message filtering feature --- metagpt/roles/engineer.py | 13 +++++++++---- metagpt/roles/qa_engineer.py | 9 +++++++-- metagpt/roles/role.py | 6 ++++-- requirements.txt | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index ba622429b..62e1e92d2 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -21,7 +21,12 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import CodeParser, get_class_name, get_object_name +from metagpt.utils.common import ( + CodeParser, + any_to_str_set, + get_class_name, + get_object_name, +) from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -102,7 +107,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] + msg = self._rc.memory.get_by_action(get_class_name(WriteDesign))[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -130,7 +135,7 @@ async def _act_mp(self) -> Message: todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), + context=self._rc.memory.get_by_actions(any_to_str_set([WriteTasks, WriteDesign])), filename=todo, ) todo_coros.append(todo_coro) @@ -185,7 +190,7 @@ async def _act_sp_precision(self) -> Message: TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + msg = self._rc.memory.get_by_actions(any_to_str_set([WriteDesign, WriteTasks, WriteCode])) for m in msg: context.append(m.content) context_str = "\n".join(context) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 760b65736..38fb5a24b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -22,7 +22,12 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import CodeParser, any_to_str_set, parse_recipient +from metagpt.utils.common import ( + CodeParser, + any_to_str_set, + get_class_name, + parse_recipient, +) from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -50,7 +55,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] + msg = self._rc.memory.get_by_action(get_class_name(WriteDesign))[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f3e11b294..b8be309bb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -217,9 +217,11 @@ async def _act(self) -> Message: async def _observe(self) -> int: """Prepare new messages for processing from the message buffer and other sources.""" # Read unprocessed messages from the msg buffer. - self._rc.news = self._rc.msg_buffer.pop_all() + news = self._rc.msg_buffer.pop_all() # Store the read messages in your own memory to prevent duplicate processing. - self._rc.memory.add_batch(self._rc.news) + self._rc.memory.add_batch(news) + # Filter out messages of interest. + self._rc.news = [n for n in news if n.cause_by in self._rc.watch] # Design Rules: # If you need to further categorize Message objects, you can do so using the Message.set_meta function. diff --git a/requirements.txt b/requirements.txt index 24a2d94c3..c3b909e77 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ langchain==0.0.231 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 -openai +openai==0.28.1 openpyxl beautifulsoup4==4.12.2 pandas==2.0.3 From 1be1bb56e3558a257e331e759cd71aa0a7b755eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 9 Nov 2023 11:52:34 +0800 Subject: [PATCH 087/232] fixbug: utilize the new message filtering feature --- metagpt/roles/engineer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 62e1e92d2..a108fa4f1 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -239,8 +239,9 @@ async def _observe(self) -> int: if not message.cause_by == get_class_name(WriteTasks): continue self.todos = self.parse_tasks(message) + return 1 - return ret + return 0 async def _think(self) -> None: # In asynchronous scenarios, first check if the required messages are ready. From a3cb2b4fdcaa47f48704cbfead054b89f79cd3b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 15:27:27 +0800 Subject: [PATCH 088/232] feat: replace get_class_name and get_object_name --- metagpt/roles/engineer.py | 23 +++++++++-------------- metagpt/roles/researcher.py | 10 ++++------ metagpt/roles/role.py | 10 +++++----- metagpt/roles/seacher.py | 6 +++--- metagpt/roles/sk_agent.py | 4 ++-- 5 files changed, 23 insertions(+), 30 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index a108fa4f1..70dce41b1 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -21,12 +21,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import ( - CodeParser, - any_to_str_set, - get_class_name, - get_object_name, -) +from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -107,7 +102,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(get_class_name(WriteDesign))[-1] + msg = self._rc.memory.get_by_action(any_to_str(WriteDesign))[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -146,13 +141,13 @@ async def _act_mp(self) -> Message: logger.info(todo) logger.info(code_rsp) # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=get_object_name(self._rc.todo)) + msg = Message(content=code_rsp, role=self.profile, cause_by=any_to_str(self._rc.todo)) self._rc.memory.add(msg) self.publish_message(msg) del self.todos[0] logger.info(f"Done {self.get_workspace()} generating.") - msg = Message(content="all done.", role=self.profile, cause_by=get_object_name(self._rc.todo)) + msg = Message(content="all done.", role=self.profile, cause_by=any_to_str(self._rc.todo)) return msg async def _act_sp(self) -> Message: @@ -163,7 +158,7 @@ async def _act_sp(self) -> Message: # logger.info(code_rsp) # code = self.parse_code(code_rsp) file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=get_object_name(self._rc.todo)) + msg = Message(content=code, role=self.profile, cause_by=any_to_str(self._rc.todo)) self._rc.memory.add(msg) self.publish_message(msg) @@ -174,7 +169,7 @@ async def _act_sp(self) -> Message: msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, - cause_by=get_object_name(self._rc.todo), + cause_by=any_to_str(self._rc.todo), send_to="Edward", ) return msg @@ -216,7 +211,7 @@ async def _act_sp_precision(self) -> Message: msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, - cause_by=get_object_name(self._rc.todo), + cause_by=any_to_str(self._rc.todo), send_to="Edward", ) return msg @@ -236,7 +231,7 @@ async def _observe(self) -> int: # Parse task lists for message in self._rc.news: - if not message.cause_by == get_class_name(WriteTasks): + if not message.cause_by == any_to_str(WriteTasks): continue self.todos = self.parse_tasks(message) return 1 @@ -245,7 +240,7 @@ async def _observe(self) -> int: async def _think(self) -> None: # In asynchronous scenarios, first check if the required messages are ready. - filters = {get_class_name(WriteTasks)} + filters = {any_to_str(WriteTasks)} msgs = self._rc.memory.get_by_actions(filters) if not msgs: self._rc.todo = None diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 4ec6f31e1..8d5e43fab 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -15,7 +15,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import get_object_name +from metagpt.utils.common import any_to_str class Report(BaseModel): @@ -64,21 +64,19 @@ async def _act(self) -> Message: research_system_text = get_research_system_text(topic, self.language) if isinstance(todo, CollectLinks): links = await todo.run(topic, 4, 4) - ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=get_object_name(todo)) + ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=any_to_str(todo)) elif isinstance(todo, WebBrowseAndSummarize): links = instruct_content.links todos = (todo.run(*url, query=query, system_text=research_system_text) for (query, url) in links.items()) summaries = await asyncio.gather(*todos) summaries = list((url, summary) for i in summaries for (url, summary) in i.items() if summary) - ret = Message( - "", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=get_object_name(todo) - ) + ret = Message("", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=any_to_str(todo)) else: summaries = instruct_content.summaries summary_text = "\n---\n".join(f"url: {url}\nsummary: {summary}" for (url, summary) in summaries) content = await self._rc.todo.run(topic, summary_text, system_text=research_system_text) ret = Message( - "", Report(topic=topic, content=content), role=self.profile, cause_by=get_object_name(self._rc.todo) + "", Report(topic=topic, content=content), role=self.profile, cause_by=any_to_str(self._rc.todo) ) self._rc.memory.add(ret) return ret diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b8be309bb..90e85186b 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -30,7 +30,7 @@ from metagpt.logs import logger from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message, MessageQueue -from metagpt.utils.common import get_class_name, get_object_name +from metagpt.utils.common import any_to_str PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -116,7 +116,7 @@ def __init__(self, name="", profile="", goal="", constraints="", desc=""): self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() - self._subscription = {get_object_name(self)} + self._subscription = {any_to_str(self)} if name: self._subscription.add(name) @@ -137,7 +137,7 @@ def _init_actions(self, actions): def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors in private message buffer""" - tags = {get_class_name(t) for t in actions} + tags = {any_to_str(t) for t in actions} self._rc.watch.update(tags) # check RoleContext after adding watch actions self._rc.check(self._role_id) @@ -207,10 +207,10 @@ async def _act(self) -> Message: content=response.content, instruct_content=response.instruct_content, role=self.profile, - cause_by=get_object_name(self._rc.todo), + cause_by=any_to_str(self._rc.todo), ) else: - msg = Message(content=response, role=self.profile, cause_by=get_object_name(self._rc.todo)) + msg = Message(content=response, role=self.profile, cause_by=any_to_str(self._rc.todo)) return msg diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index d0b841f39..a37143196 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -12,7 +12,7 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.tools import SearchEngineType -from metagpt.utils.common import get_object_name +from metagpt.utils.common import any_to_str class Searcher(Role): @@ -64,10 +64,10 @@ async def _act_sp(self) -> Message: content=response.content, instruct_content=response.instruct_content, role=self.profile, - cause_by=get_object_name(self._rc.todo), + cause_by=any_to_str(self._rc.todo), ) else: - msg = Message(content=response, role=self.profile, cause_by=get_object_name(self._rc.todo)) + msg = Message(content=response, role=self.profile, cause_by=any_to_str(self._rc.todo)) self._rc.memory.add(msg) return msg diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index 5b8d333bd..bb923caf2 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -17,7 +17,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import get_object_name +from metagpt.utils.common import any_to_str from metagpt.utils.make_sk_kernel import make_sk_kernel @@ -74,7 +74,7 @@ async def _act(self) -> Message: result = (await self.plan.invoke_async()).result logger.info(result) - msg = Message(content=result, role=self.profile, cause_by=get_object_name(self._rc.todo)) + msg = Message(content=result, role=self.profile, cause_by=any_to_str(self._rc.todo)) self._rc.memory.add(msg) self.publish_message(msg) return msg From d36b4e2088c4a2e48c4f1ab63fc9c41c163bbaf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 15:41:27 +0800 Subject: [PATCH 089/232] refactor: replace obj with role --- metagpt/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index e9a5c6467..b3c296dac 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -61,9 +61,9 @@ def publish_message(self, message: Message) -> bool: logger.info(f"publish_message: {message.dump()}") found = False # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 - for obj, subscription in self.consumers.items(): + for role, subscription in self.consumers.items(): if is_subscribed(message, subscription): - obj.put_message(message) + role.put_message(message) found = True if not found: logger.warning(f"Message no recipients: {message.dump()}") From 3c38c5c41678f64a43f430ab215618567b98471b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 15:55:33 +0800 Subject: [PATCH 090/232] refactor: get_class_name --- metagpt/actions/write_code.py | 4 ++-- metagpt/memory/memory.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index be8690314..aeaa10aec 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -14,7 +14,7 @@ from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.common import CodeParser, get_class_name +from metagpt.utils.common import CodeParser, any_to_str PROMPT_TEMPLATE = """ NOTICE @@ -58,7 +58,7 @@ def _save(self, context, filename, code): if self._is_invalid(filename): return - design = [i for i in context if i.cause_by == get_class_name(WriteDesign)][0] + design = [i for i in context if i.cause_by == any_to_str(WriteDesign)][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) ws_path = WORKSPACE_ROOT / ws_name diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 84289091f..9d526420f 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -10,6 +10,7 @@ from typing import Iterable, Set from metagpt.schema import Message +from metagpt.utils.common import any_to_str, any_to_str_set class Memory: @@ -73,14 +74,16 @@ def find_news(self, observed: list[Message], k=0) -> list[Message]: news.append(i) return news - def get_by_action(self, action: str) -> list[Message]: + def get_by_action(self, action) -> list[Message]: """Return all messages triggered by a specified Action""" - return self.index[action] + idx = any_to_str(action) + return self.index[idx] - def get_by_actions(self, actions: Set[str]) -> list[Message]: + def get_by_actions(self, actions: Set) -> list[Message]: """Return all messages triggered by specified Actions""" + idxs = any_to_str_set(actions) rsp = [] - for action in actions: + for action in idxs: if action not in self.index: continue rsp += self.index[action] From 710bc40b0ab49e1e5c9331a7175487aab68b9db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 15:58:47 +0800 Subject: [PATCH 091/232] refactor: get_class_name --- metagpt/memory/memory.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 9d526420f..2f4c9d20b 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -10,7 +10,6 @@ from typing import Iterable, Set from metagpt.schema import Message -from metagpt.utils.common import any_to_str, any_to_str_set class Memory: @@ -76,14 +75,12 @@ def find_news(self, observed: list[Message], k=0) -> list[Message]: def get_by_action(self, action) -> list[Message]: """Return all messages triggered by a specified Action""" - idx = any_to_str(action) - return self.index[idx] + return self.index[action] def get_by_actions(self, actions: Set) -> list[Message]: """Return all messages triggered by specified Actions""" - idxs = any_to_str_set(actions) rsp = [] - for action in idxs: + for action in actions: if action not in self.index: continue rsp += self.index[action] From 60bad1830482b69dfb4ac92b128f172d19242e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 15:59:50 +0800 Subject: [PATCH 092/232] refactor: get_class_name --- metagpt/memory/memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 2f4c9d20b..71d999049 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -73,7 +73,7 @@ def find_news(self, observed: list[Message], k=0) -> list[Message]: news.append(i) return news - def get_by_action(self, action) -> list[Message]: + def get_by_action(self, action: str) -> list[Message]: """Return all messages triggered by a specified Action""" return self.index[action] From 83a5e03b72168714c277633e53e2f16dc0f57345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:05:12 +0800 Subject: [PATCH 093/232] refactor: get_class_name --- metagpt/roles/engineer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 70dce41b1..742e00cc8 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -81,7 +81,7 @@ def __init__( self.use_code_review = use_code_review if self.use_code_review: self._init_actions([WriteCode, WriteCodeReview]) - self._watch([WriteTasks, WriteDesign]) + self._watch([WriteTasks]) self.todos = [] self.n_borg = n_borg From a61f3f80e97a2265120d15195036fbb8ccf4b370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:15:07 +0800 Subject: [PATCH 094/232] refactor: get_by_action(s) --- examples/debate.py | 5 ++--- metagpt/memory/memory.py | 9 ++++++--- metagpt/roles/engineer.py | 11 +++++------ metagpt/roles/qa_engineer.py | 11 +++-------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index 8f5012d66..630f78cd8 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -15,7 +15,6 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.software_company import SoftwareCompany -from metagpt.utils.common import any_to_str_set class ShoutOut(Action): @@ -66,7 +65,7 @@ async def _observe(self) -> int: async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions(any_to_str_set([ShoutOut])) + msg_history = self._rc.memory.get_by_actions([ShoutOut]) context = [] for m in msg_history: context.append(str(m)) @@ -108,7 +107,7 @@ async def _observe(self) -> int: async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - msg_history = self._rc.memory.get_by_actions(any_to_str_set([BossRequirement, ShoutOut])) + msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) context = [] for m in msg_history: context.append(str(m)) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 71d999049..53b65fcf7 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -10,6 +10,7 @@ from typing import Iterable, Set from metagpt.schema import Message +from metagpt.utils.common import any_to_str, any_to_str_set class Memory: @@ -73,14 +74,16 @@ def find_news(self, observed: list[Message], k=0) -> list[Message]: news.append(i) return news - def get_by_action(self, action: str) -> list[Message]: + def get_by_action(self, action) -> list[Message]: """Return all messages triggered by a specified Action""" - return self.index[action] + index = any_to_str(action) + return self.index[index] def get_by_actions(self, actions: Set) -> list[Message]: """Return all messages triggered by specified Actions""" rsp = [] - for action in actions: + indices = any_to_str_set(actions) + for action in indices: if action not in self.index: continue rsp += self.index[action] diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 742e00cc8..960f9c0f3 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -21,7 +21,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set +from metagpt.utils.common import CodeParser, any_to_str from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -102,7 +102,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(any_to_str(WriteDesign))[-1] + msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -130,7 +130,7 @@ async def _act_mp(self) -> Message: todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions(any_to_str_set([WriteTasks, WriteDesign])), + context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo, ) todo_coros.append(todo_coro) @@ -185,7 +185,7 @@ async def _act_sp_precision(self) -> Message: TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ context = [] - msg = self._rc.memory.get_by_actions(any_to_str_set([WriteDesign, WriteTasks, WriteCode])) + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) context_str = "\n".join(context) @@ -240,8 +240,7 @@ async def _observe(self) -> int: async def _think(self) -> None: # In asynchronous scenarios, first check if the required messages are ready. - filters = {any_to_str(WriteTasks)} - msgs = self._rc.memory.get_by_actions(filters) + msgs = self._rc.memory.get_by_actions({WriteTasks}) if not msgs: self._rc.todo = None return diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 38fb5a24b..9495e1a12 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -22,12 +22,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import ( - CodeParser, - any_to_str_set, - get_class_name, - parse_recipient, -) +from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set, parse_recipient from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -55,7 +50,7 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(get_class_name(WriteDesign))[-1] + msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -104,7 +99,7 @@ async def _write_test(self, message: Message) -> None: msg = Message( content=str(file_info), role=self.profile, - cause_by=WriteTest, + cause_by=any_to_str(WriteTest), sent_from=self.profile, send_to=self.profile, ) From bb8e2467ea6d0405cf42da1850b85962b3570915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:22:02 +0800 Subject: [PATCH 095/232] refactor: cause_by --- metagpt/roles/qa_engineer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 9495e1a12..0f932ebfb 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -133,7 +133,9 @@ async def _run_code(self, msg): recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message(content=content, role=self.profile, cause_by=RunCode, sent_from=self.profile, send_to=recipient) + msg = Message( + content=content, role=self.profile, cause_by=any_to_str(RunCode), sent_from=self.profile, send_to=recipient + ) self.publish_message(msg) async def _debug_error(self, msg): @@ -145,7 +147,7 @@ async def _debug_error(self, msg): msg = Message( content=file_info, role=self.profile, - cause_by=DebugError, + cause_by=any_to_str(DebugError), sent_from=self.profile, send_to=recipient, ) @@ -163,7 +165,7 @@ async def _act(self) -> Message: result_msg = Message( content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", role=self.profile, - cause_by=WriteTest, + cause_by=any_to_str(WriteTest), sent_from=self.profile, ) return result_msg @@ -187,7 +189,7 @@ async def _act(self) -> Message: result_msg = Message( content=f"Round {self.test_round} of tests done", role=self.profile, - cause_by=WriteTest, + cause_by=any_to_str(WriteTest), sent_from=self.profile, ) return result_msg From fc63cdf4df1fdf6e4bf1ed3ecff8a58a9ad0a098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:24:09 +0800 Subject: [PATCH 096/232] refactor: cause_by --- metagpt/roles/role.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 90e85186b..a0a35bdc2 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -116,9 +116,7 @@ def __init__(self, name="", profile="", goal="", constraints="", desc=""): self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() - self._subscription = {any_to_str(self)} - if name: - self._subscription.add(name) + self._subscription = {any_to_str(self), name} if name else {any_to_str(self)} def _reset(self): self._states = [] From 9ebd1d1bbb3ed55564cde68396624120f6d9dec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:28:29 +0800 Subject: [PATCH 097/232] refactor: notation --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index a0a35bdc2..75e41d4ae 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -134,7 +134,7 @@ def _init_actions(self, actions): self._states.append(f"{idx}. {action}") def _watch(self, actions: Iterable[Type[Action]]): - """Listen to the corresponding behaviors in private message buffer""" + """Watch Actions of interest. Role will select Messages caused by these Actions from its personal message buffer during _observe.""" tags = {any_to_str(t) for t in actions} self._rc.watch.update(tags) # check RoleContext after adding watch actions From b1a14d057a6c5af98c624881db0590928bd04b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:29:26 +0800 Subject: [PATCH 098/232] refactor: notation --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 75e41d4ae..ec6d71684 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -141,7 +141,7 @@ def _watch(self, actions: Iterable[Type[Action]]): self._rc.check(self._role_id) def subscribe(self, tags: Set[str]): - """Listen to the corresponding behaviors""" + """Used to receive Messages with certain tags from the environment. Message will be put into personal message buffer to be further processed in _observe. By default, a Role subscribes Messages with a tag of its own name or profile.""" self._subscription = tags if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 self._rc.env.set_subscription(self, self._subscription) From e8eeb6cda97c26ac0b55e854c5d67910a6f218da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:29:41 +0800 Subject: [PATCH 099/232] refactor: notation --- metagpt/roles/role.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index ec6d71684..4201b0f92 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -134,14 +134,17 @@ def _init_actions(self, actions): self._states.append(f"{idx}. {action}") def _watch(self, actions: Iterable[Type[Action]]): - """Watch Actions of interest. Role will select Messages caused by these Actions from its personal message buffer during _observe.""" + """Watch Actions of interest. Role will select Messages caused by these Actions from its personal message + buffer during _observe.""" tags = {any_to_str(t) for t in actions} self._rc.watch.update(tags) # check RoleContext after adding watch actions self._rc.check(self._role_id) def subscribe(self, tags: Set[str]): - """Used to receive Messages with certain tags from the environment. Message will be put into personal message buffer to be further processed in _observe. By default, a Role subscribes Messages with a tag of its own name or profile.""" + """Used to receive Messages with certain tags from the environment. Message will be put into personal message + buffer to be further processed in _observe. By default, a Role subscribes Messages with a tag of its own name + or profile.""" self._subscription = tags if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 self._rc.env.set_subscription(self, self._subscription) From efe6ead27c263afc2a39b2ba7b0a65c6376458bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:30:23 +0800 Subject: [PATCH 100/232] refactor: notation --- metagpt/roles/role.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 4201b0f92..ccad0b018 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -135,7 +135,8 @@ def _init_actions(self, actions): def _watch(self, actions: Iterable[Type[Action]]): """Watch Actions of interest. Role will select Messages caused by these Actions from its personal message - buffer during _observe.""" + buffer during _observe. + """ tags = {any_to_str(t) for t in actions} self._rc.watch.update(tags) # check RoleContext after adding watch actions @@ -144,7 +145,8 @@ def _watch(self, actions: Iterable[Type[Action]]): def subscribe(self, tags: Set[str]): """Used to receive Messages with certain tags from the environment. Message will be put into personal message buffer to be further processed in _observe. By default, a Role subscribes Messages with a tag of its own name - or profile.""" + or profile. + """ self._subscription = tags if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 self._rc.env.set_subscription(self, self._subscription) From 44aa1dd563d04957f084b5b4a91c7105533eb002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:48:34 +0800 Subject: [PATCH 101/232] refactor: cause_by --- examples/debate.py | 5 ++++- metagpt/roles/engineer.py | 10 +++++----- metagpt/roles/qa_engineer.py | 14 ++++++-------- metagpt/roles/researcher.py | 9 +++------ metagpt/roles/role.py | 4 ++-- metagpt/roles/seacher.py | 5 ++--- 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/examples/debate.py b/examples/debate.py index 630f78cd8..597b44e8d 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -15,6 +15,7 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.software_company import SoftwareCompany +from metagpt.utils.common import any_to_str class ShoutOut(Action): @@ -101,7 +102,9 @@ async def _observe(self) -> int: await super()._observe() # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, # disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.cause_by == BossRequirement or msg.send_to == {self.name}] + self._rc.news = [ + msg for msg in self._rc.news if msg.cause_by == any_to_str(BossRequirement) or msg.send_to == {self.name} + ] return len(self._rc.news) async def _act(self) -> Message: diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 960f9c0f3..535a1e27f 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -141,13 +141,13 @@ async def _act_mp(self) -> Message: logger.info(todo) logger.info(code_rsp) # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=any_to_str(self._rc.todo)) + msg = Message(content=code_rsp, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) self.publish_message(msg) del self.todos[0] logger.info(f"Done {self.get_workspace()} generating.") - msg = Message(content="all done.", role=self.profile, cause_by=any_to_str(self._rc.todo)) + msg = Message(content="all done.", role=self.profile, cause_by=self._rc.todo) return msg async def _act_sp(self) -> Message: @@ -158,7 +158,7 @@ async def _act_sp(self) -> Message: # logger.info(code_rsp) # code = self.parse_code(code_rsp) file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=any_to_str(self._rc.todo)) + msg = Message(content=code, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) self.publish_message(msg) @@ -169,7 +169,7 @@ async def _act_sp(self) -> Message: msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, - cause_by=any_to_str(self._rc.todo), + cause_by=self._rc.todo, send_to="Edward", ) return msg @@ -211,7 +211,7 @@ async def _act_sp_precision(self) -> Message: msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, - cause_by=any_to_str(self._rc.todo), + cause_by=self._rc.todo, send_to="Edward", ) return msg diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 0f932ebfb..760b65736 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -22,7 +22,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set, parse_recipient +from metagpt.utils.common import CodeParser, any_to_str_set, parse_recipient from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP @@ -99,7 +99,7 @@ async def _write_test(self, message: Message) -> None: msg = Message( content=str(file_info), role=self.profile, - cause_by=any_to_str(WriteTest), + cause_by=WriteTest, sent_from=self.profile, send_to=self.profile, ) @@ -133,9 +133,7 @@ async def _run_code(self, msg): recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message( - content=content, role=self.profile, cause_by=any_to_str(RunCode), sent_from=self.profile, send_to=recipient - ) + msg = Message(content=content, role=self.profile, cause_by=RunCode, sent_from=self.profile, send_to=recipient) self.publish_message(msg) async def _debug_error(self, msg): @@ -147,7 +145,7 @@ async def _debug_error(self, msg): msg = Message( content=file_info, role=self.profile, - cause_by=any_to_str(DebugError), + cause_by=DebugError, sent_from=self.profile, send_to=recipient, ) @@ -165,7 +163,7 @@ async def _act(self) -> Message: result_msg = Message( content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", role=self.profile, - cause_by=any_to_str(WriteTest), + cause_by=WriteTest, sent_from=self.profile, ) return result_msg @@ -189,7 +187,7 @@ async def _act(self) -> Message: result_msg = Message( content=f"Round {self.test_round} of tests done", role=self.profile, - cause_by=any_to_str(WriteTest), + cause_by=WriteTest, sent_from=self.profile, ) return result_msg diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 8d5e43fab..29889b8ec 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -15,7 +15,6 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import any_to_str class Report(BaseModel): @@ -64,20 +63,18 @@ async def _act(self) -> Message: research_system_text = get_research_system_text(topic, self.language) if isinstance(todo, CollectLinks): links = await todo.run(topic, 4, 4) - ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=any_to_str(todo)) + ret = Message("", Report(topic=topic, links=links), role=self.profile, cause_by=todo) elif isinstance(todo, WebBrowseAndSummarize): links = instruct_content.links todos = (todo.run(*url, query=query, system_text=research_system_text) for (query, url) in links.items()) summaries = await asyncio.gather(*todos) summaries = list((url, summary) for i in summaries for (url, summary) in i.items() if summary) - ret = Message("", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=any_to_str(todo)) + ret = Message("", Report(topic=topic, summaries=summaries), role=self.profile, cause_by=todo) else: summaries = instruct_content.summaries summary_text = "\n---\n".join(f"url: {url}\nsummary: {summary}" for (url, summary) in summaries) content = await self._rc.todo.run(topic, summary_text, system_text=research_system_text) - ret = Message( - "", Report(topic=topic, content=content), role=self.profile, cause_by=any_to_str(self._rc.todo) - ) + ret = Message("", Report(topic=topic, content=content), role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(ret) return ret diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index ccad0b018..5c512b0f0 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -210,10 +210,10 @@ async def _act(self) -> Message: content=response.content, instruct_content=response.instruct_content, role=self.profile, - cause_by=any_to_str(self._rc.todo), + cause_by=self._rc.todo, ) else: - msg = Message(content=response, role=self.profile, cause_by=any_to_str(self._rc.todo)) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) return msg diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index a37143196..587698d1d 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -12,7 +12,6 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.tools import SearchEngineType -from metagpt.utils.common import any_to_str class Searcher(Role): @@ -64,10 +63,10 @@ async def _act_sp(self) -> Message: content=response.content, instruct_content=response.instruct_content, role=self.profile, - cause_by=any_to_str(self._rc.todo), + cause_by=self._rc.todo, ) else: - msg = Message(content=response, role=self.profile, cause_by=any_to_str(self._rc.todo)) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) return msg From 7fb33fd890a3a19ab46f60d2ac9df152e75278b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 16:56:50 +0800 Subject: [PATCH 102/232] refactor: cause_by --- metagpt/roles/sk_agent.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index bb923caf2..15b18dd3e 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -4,9 +4,8 @@ @Time : 2023/9/13 12:23 @Author : femto Zheng @File : sk_agent.py -@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data - type of the `cause_by` value in the `Message` to a string, and utilize the new message distribution - feature for message filtering. +@Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message + distribution feature for message filtering. """ from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner @@ -17,7 +16,6 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import any_to_str from metagpt.utils.make_sk_kernel import make_sk_kernel @@ -74,7 +72,7 @@ async def _act(self) -> Message: result = (await self.plan.invoke_async()).result logger.info(result) - msg = Message(content=result, role=self.profile, cause_by=any_to_str(self._rc.todo)) + msg = Message(content=result, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) self.publish_message(msg) return msg From d9a7443e5a1b67dc5286fcd1d56a8ba8b540ec90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 17:03:24 +0800 Subject: [PATCH 103/232] refactor: notation --- metagpt/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 63fe41232..82a0117ef 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -58,7 +58,7 @@ def __init__( :param instruct_content: Message content struct. :param cause_by: Message producer :param sent_from: Message route info tells who sent this message. - :param send_to: Labels for the consumer to filter its subscribed messages. + :param send_to: Specifies the target recipient or consumer for message delivery in the environment. :param role: Message meta info tells who sent this message. """ super().__init__( From 282a86bfa7b28e44c27b425432e87b1bf1c0c37c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 10 Nov 2023 17:14:58 +0800 Subject: [PATCH 104/232] refactor: unit tests --- tests/metagpt/test_role.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 447de7ee5..8fac2503c 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -60,7 +60,7 @@ class Input(BaseModel): name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, desc=seed.desc ) role.subscribe({seed.subscription}) - assert role._rc.watch == {seed.subscription} + assert role._rc.watch == set({}) assert role.name == seed.name assert role.profile == seed.profile assert role._setting.goal == seed.goal @@ -69,7 +69,7 @@ class Input(BaseModel): assert role.is_idle env = Environment() env.add_role(role) - assert env.get_subscribed_tags(role) == {seed.subscription} + assert env.get_subscription(role) == {seed.subscription} env.publish_message(Message(content="test", msg_to=seed.subscription)) assert not role.is_idle while not env.is_idle: @@ -82,19 +82,19 @@ class Input(BaseModel): assert role.is_idle tag = uuid.uuid4().hex role.subscribe({tag}) - assert env.get_subscribed_tags(role) == {seed.subscription, tag} + assert env.get_subscription(role) == {tag} @pytest.mark.asyncio async def test_msg_to(): - m = Message(content="a", msg_to=["a", MockRole, Message]) - assert m.msg_to == {"a", get_class_name(MockRole), get_class_name(Message)} + m = Message(content="a", send_to=["a", MockRole, Message]) + assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) - m = Message(content="a", cause_by=MockAction, msg_to={"a", MockRole, Message}) - assert m.msg_to == {"a", get_class_name(MockRole), get_class_name(Message), get_class_name(MockAction)} + m = Message(content="a", cause_by=MockAction, send_to={"a", MockRole, Message}) + assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) - m = Message(content="a", msg_to=("a", MockRole, Message)) - assert m.msg_to == {"a", get_class_name(MockRole), get_class_name(Message)} + m = Message(content="a", send_to=("a", MockRole, Message)) + assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) if __name__ == "__main__": From 23749212bfe3b705f0f47758c4dca42efe337eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 12 Nov 2023 17:41:51 +0800 Subject: [PATCH 105/232] refactor: rename --- metagpt/roles/role.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 5c512b0f0..a8280cecf 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -255,16 +255,16 @@ async def _react(self) -> Message: logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") return await self._act() - async def run(self, test_message=None): + async def run(self, with_message=None): """Observe, and think and act based on the results of the observation""" - if test_message: # For test + if with_message: # For test msg = None - if isinstance(test_message, str): - msg = Message(test_message) - elif isinstance(test_message, Message): - msg = test_message - elif isinstance(test_message, list): - msg = Message("\n".join(test_message)) + if isinstance(with_message, str): + msg = Message(with_message) + elif isinstance(with_message, Message): + msg = with_message + elif isinstance(with_message, list): + msg = Message("\n".join(with_message)) self.put_message(msg) if not await self._observe(): From 962109bd119ba7cdde0f4ef7c33b3830c8ff9bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 13 Nov 2023 16:26:24 +0800 Subject: [PATCH 106/232] refactor: notation --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index a8280cecf..2e3bcbbd5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -257,7 +257,7 @@ async def _react(self) -> Message: async def run(self, with_message=None): """Observe, and think and act based on the results of the observation""" - if with_message: # For test + if with_message: msg = None if isinstance(with_message, str): msg = Message(with_message) From c2ffee61e6730d7dcf31168dd6f0cd713d92a98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 13 Nov 2023 20:45:05 +0800 Subject: [PATCH 107/232] refactor: remove useless code --- metagpt/roles/engineer.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 535a1e27f..d23d23d55 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -237,12 +237,3 @@ async def _observe(self) -> int: return 1 return 0 - - async def _think(self) -> None: - # In asynchronous scenarios, first check if the required messages are ready. - msgs = self._rc.memory.get_by_actions({WriteTasks}) - if not msgs: - self._rc.todo = None - return - - await super(Engineer, self)._think() From 0cf6ec1a93e40ad33ebb46b4060e10a312138253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:27:16 +0800 Subject: [PATCH 108/232] feat: +git repo --- metagpt/utils/git_repository.py | 110 +++++++++++++++++++++ tests/metagpt/utils/test_git_repository.py | 79 +++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 metagpt/utils/git_repository.py create mode 100644 tests/metagpt/utils/test_git_repository.py diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py new file mode 100644 index 000000000..fd9794a80 --- /dev/null +++ b/metagpt/utils/git_repository.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: Git repository management +""" +from __future__ import annotations + +import shutil +from enum import Enum +from pathlib import Path +from typing import Dict + +from git.repo import Repo +from git.repo.fun import is_git_dir + +from metagpt.const import WORKSPACE_ROOT + + +class ChangeType(Enum): + ADDED = "A" # File was added + COPIED = "C" # File was copied + DELETED = "D" # File was deleted + RENAMED = "R" # File was renamed + MODIFIED = "M" # File was modified + TYPE_CHANGED = "T" # Type of the file was changed + UNTRACTED = "U" # File is untracked (not added to version control) + + +class GitRepository: + def __init__(self, local_path=None, auto_init=True): + self._repository = None + if local_path: + self.open(local_path=local_path, auto_init=auto_init) + + def open(self, local_path: Path, auto_init=False): + if self.is_git_dir(local_path): + self._repository = Repo(local_path) + return + if not auto_init: + return + local_path.mkdir(parents=True, exist_ok=True) + return self._init(local_path) + + def _init(self, local_path: Path): + self._repository = Repo.init(path=local_path) + + def add_change(self, files: Dict): + if not self.is_valid or not files: + return + + for k, v in files.items(): + self._repository.index.remove(k) if v is ChangeType.DELETED else self._repository.index.add([k]) + + def commit(self, comments): + if self.is_valid: + self._repository.index.commit(comments) + + def delete_repository(self): + # Delete the repository directory + if self.is_valid: + shutil.rmtree(self._repository.working_dir) + + @property + def changed_files(self) -> Dict[str, str]: + files = {i: ChangeType.UNTRACTED for i in self._repository.untracked_files} + changed_files = {f.a_path: ChangeType(f.change_type) for f in self._repository.index.diff(None)} + files.update(changed_files) + return files + + @staticmethod + def is_git_dir(local_path): + git_dir = local_path / ".git" + if git_dir.exists() and is_git_dir(git_dir): + return True + return False + + @property + def is_valid(self): + return bool(self._repository) + + @property + def status(self) -> str: + if not self.is_valid: + return "" + return self._repository.git.status() + + @property + def workdir(self) -> Path | None: + if not self.is_valid: + return None + return Path(self._repository.working_dir) + + +if __name__ == "__main__": + path = WORKSPACE_ROOT / "git" + path.mkdir(exist_ok=True, parents=True) + + repo = GitRepository() + repo.open(path, auto_init=True) + + changes = repo.changed_files + print(changes) + repo.add_change(changes) + print(repo.status) + repo.commit("test") + print(repo.status) + repo.delete_repository() diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py new file mode 100644 index 000000000..2e15f44f9 --- /dev/null +++ b/tests/metagpt/utils/test_git_repository.py @@ -0,0 +1,79 @@ +import shutil +from pathlib import Path + +import aiofiles +import pytest + +from metagpt.utils.git_repository import GitRepository + + +async def mock_file(filename, content=""): + async with aiofiles.open(str(filename), mode="w") as file: + await file.write(content) + + +@pytest.mark.asyncio +async def test_git(): + local_path = Path(__file__).parent / "git" + if local_path.exists(): + shutil.rmtree(local_path) + assert not local_path.exists() + repo = GitRepository(local_path=local_path, auto_init=True) + assert local_path.exists() + assert local_path == repo.workdir + assert not repo.changed_files + + await mock_file(local_path / "a.txt") + await mock_file(local_path / "b.txt") + subdir = local_path / "subdir" + subdir.mkdir(parents=True, exist_ok=True) + await mock_file(subdir / "c.txt") + + assert len(repo.changed_files) == 3 + repo.add_change(repo.changed_files) + repo.commit("commit1") + assert not repo.changed_files + + await mock_file(local_path / "a.txt", "tests") + await mock_file(subdir / "d.txt") + rmfile = local_path / "b.txt" + rmfile.unlink() + assert repo.status + + assert len(repo.changed_files) == 3 + repo.add_change(repo.changed_files) + repo.commit("commit2") + assert not repo.changed_files + + assert repo.status + + repo.delete_repository() + assert not local_path.exists() + + +@pytest.mark.asyncio +async def test_git1(): + local_path = Path(__file__).parent / "git1" + if local_path.exists(): + shutil.rmtree(local_path) + assert not local_path.exists() + repo = GitRepository(local_path=local_path, auto_init=True) + assert local_path.exists() + assert local_path == repo.workdir + assert not repo.changed_files + + await mock_file(local_path / "a.txt") + await mock_file(local_path / "b.txt") + subdir = local_path / "subdir" + subdir.mkdir(parents=True, exist_ok=True) + await mock_file(subdir / "c.txt") + + repo1 = GitRepository(local_path=local_path, auto_init=False) + assert repo1.changed_files + + repo1.delete_repository() + assert not local_path.exists() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 9c5f7c76719e07845da74c7ef915388b44722433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:33:46 +0800 Subject: [PATCH 109/232] feat: +annotation --- metagpt/utils/git_repository.py | 51 +++++++++++++++++++++- tests/metagpt/utils/test_git_repository.py | 25 ++++------- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index fd9794a80..c5b510612 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -30,12 +30,31 @@ class ChangeType(Enum): class GitRepository: + """A class representing a Git repository. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + + Attributes: + _repository (Repo): The GitPython `Repo` object representing the Git repository. + """ + def __init__(self, local_path=None, auto_init=True): + """Initialize a GitRepository instance. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + """ self._repository = None if local_path: self.open(local_path=local_path, auto_init=auto_init) def open(self, local_path: Path, auto_init=False): + """Open an existing Git repository or initialize a new one if auto_init is True. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + """ if self.is_git_dir(local_path): self._repository = Repo(local_path) return @@ -45,9 +64,17 @@ def open(self, local_path: Path, auto_init=False): return self._init(local_path) def _init(self, local_path: Path): + """Initialize a new Git repository at the specified path. + + :param local_path: The local path where the new Git repository will be initialized. + """ self._repository = Repo.init(path=local_path) def add_change(self, files: Dict): + """Add or remove files from the staging area based on the provided changes. + + :param files: A dictionary where keys are file paths and values are instances of ChangeType. + """ if not self.is_valid or not files: return @@ -55,16 +82,24 @@ def add_change(self, files: Dict): self._repository.index.remove(k) if v is ChangeType.DELETED else self._repository.index.add([k]) def commit(self, comments): + """Commit the staged changes with the given comments. + + :param comments: Comments for the commit. + """ if self.is_valid: self._repository.index.commit(comments) def delete_repository(self): - # Delete the repository directory + """Delete the entire repository directory.""" if self.is_valid: shutil.rmtree(self._repository.working_dir) @property def changed_files(self) -> Dict[str, str]: + """Return a dictionary of changed files and their change types. + + :return: A dictionary where keys are file paths and values are change types. + """ files = {i: ChangeType.UNTRACTED for i in self._repository.untracked_files} changed_files = {f.a_path: ChangeType(f.change_type) for f in self._repository.index.diff(None)} files.update(changed_files) @@ -72,6 +107,11 @@ def changed_files(self) -> Dict[str, str]: @staticmethod def is_git_dir(local_path): + """Check if the specified directory is a Git repository. + + :param local_path: The local path to check. + :return: True if the directory is a Git repository, False otherwise. + """ git_dir = local_path / ".git" if git_dir.exists() and is_git_dir(git_dir): return True @@ -79,16 +119,25 @@ def is_git_dir(local_path): @property def is_valid(self): + """Check if the Git repository is valid (exists and is initialized). + + :return: True if the repository is valid, False otherwise. + """ return bool(self._repository) @property def status(self) -> str: + """Return the Git repository's status as a string.""" if not self.is_valid: return "" return self._repository.git.status() @property def workdir(self) -> Path | None: + """Return the path to the working directory of the Git repository. + + :return: The path to the working directory or None if the repository is not valid. + """ if not self.is_valid: return None return Path(self._repository.working_dir) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index 2e15f44f9..fa329a2ec 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -12,9 +12,7 @@ async def mock_file(filename, content=""): await file.write(content) -@pytest.mark.asyncio -async def test_git(): - local_path = Path(__file__).parent / "git" +async def mock_repo(local_path) -> (GitRepository, Path): if local_path.exists(): shutil.rmtree(local_path) assert not local_path.exists() @@ -28,6 +26,13 @@ async def test_git(): subdir = local_path / "subdir" subdir.mkdir(parents=True, exist_ok=True) await mock_file(subdir / "c.txt") + return repo, subdir + + +@pytest.mark.asyncio +async def test_git(): + local_path = Path(__file__).parent / "git" + repo, subdir = await mock_repo(local_path) assert len(repo.changed_files) == 3 repo.add_change(repo.changed_files) @@ -54,19 +59,7 @@ async def test_git(): @pytest.mark.asyncio async def test_git1(): local_path = Path(__file__).parent / "git1" - if local_path.exists(): - shutil.rmtree(local_path) - assert not local_path.exists() - repo = GitRepository(local_path=local_path, auto_init=True) - assert local_path.exists() - assert local_path == repo.workdir - assert not repo.changed_files - - await mock_file(local_path / "a.txt") - await mock_file(local_path / "b.txt") - subdir = local_path / "subdir" - subdir.mkdir(parents=True, exist_ok=True) - await mock_file(subdir / "c.txt") + await mock_repo(local_path) repo1 = GitRepository(local_path=local_path, auto_init=False) assert repo1.changed_files From f1fb3b3bece668590557455ae51ecf9b8f306109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:35:16 +0800 Subject: [PATCH 110/232] feat: +annotation --- tests/metagpt/utils/test_git_repository.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index fa329a2ec..0d1e3b791 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : test_git_repository.py +@Desc: Unit tests for git_repository.py +""" + import shutil from pathlib import Path From 363be23045e552d324e2946d16bc0eb29d5302f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:44:06 +0800 Subject: [PATCH 111/232] feat: +annotation --- metagpt/utils/file_repository.py | 0 metagpt/utils/git_repository.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 metagpt/utils/file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index c5b510612..1732d6a91 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -4,7 +4,7 @@ @Time : 2023/11/20 @Author : mashenquan @File : git_repository.py -@Desc: Git repository management +@Desc: Git repository management. RFC 135 2.2.3.3. """ from __future__ import annotations From af716c6c305254cfb48fda5a865616126931edd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:47:12 +0800 Subject: [PATCH 112/232] feat: +annotation --- metagpt/utils/file_repository.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 metagpt/utils/file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py deleted file mode 100644 index e69de29bb..000000000 From 990d79179f48ffc5afce0276dec1cfeb2db4ef9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 17:33:24 +0800 Subject: [PATCH 113/232] feat: archive --- metagpt/utils/file_repository.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 metagpt/utils/file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py new file mode 100644 index 000000000..e69de29bb From 9f7da1c7688f48d9a7ac2cf38c7f81cca35f7ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 17:34:44 +0800 Subject: [PATCH 114/232] feat: archive --- metagpt/utils/file_repository.py | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index e69de29bb..af787c70a 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: File repository management. RFC 135 2.2.3.2, 2.2.3.4 and 2.2.3.13. +""" +import json +from pathlib import Path +from typing import Dict, List + +import aiofiles + +from metagpt.utils.git_repository import GitRepository + + +class FileRepository: + def __init__(self, git_repo: GitRepository, relative_path: Path = "."): + self._relative_path = relative_path # Relative path based on the Git repository. + self._git_repo = git_repo + self._dependencies: Dict[str, List[str]] = {} + + async def save(self, filename: Path, content, dependencies: List[str] = None): + path_name = self.workdir / filename + with aiofiles.open(str(path_name), mode="w") as writer: + await writer.write(content) + if dependencies is not None: + await self.update_dependency(filename, dependencies) + + async def update_dependency(self, filename, dependencies: List[str]): + self._dependencies[str(filename)] = dependencies + + async def save_dependency(self): + filename = ".dependencies.json" + path_name = self.workdir / filename + data = json.dumps(self._dependencies) + with aiofiles.open(str(path_name), mode="w") as writer: + await writer.write(data) + + @property + def workdir(self): + return self._git_repo.workdir / self._relative_path From 913cfaebabc22d1130bb9cff9b8a4713b2cd72cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 17:41:16 +0800 Subject: [PATCH 115/232] feat: archive --- metagpt/utils/git_repository.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 1732d6a91..6e624c8b5 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -142,6 +142,14 @@ def workdir(self) -> Path | None: return None return Path(self._repository.working_dir) + def archive(self, comments="Archive"): + """Archive the current state of the Git repository. + + :param comments: Comments for the archive commit. + """ + self.add_change(self.changed_files) + self.commit(comments) + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" From 29003a9beb0f1ede36c1139ee8bb3815e0fdad49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 19:36:54 +0800 Subject: [PATCH 116/232] feat: +file repository --- metagpt/utils/file_repository.py | 71 +++++++++++++++++++-- metagpt/utils/git_repository.py | 9 +++ tests/metagpt/utils/test_file_repository.py | 49 ++++++++++++++ 3 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 tests/metagpt/utils/test_file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index af787c70a..d8637fe3f 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -6,38 +6,95 @@ @File : git_repository.py @Desc: File repository management. RFC 135 2.2.3.2, 2.2.3.4 and 2.2.3.13. """ +from __future__ import annotations + import json from pathlib import Path from typing import Dict, List import aiofiles -from metagpt.utils.git_repository import GitRepository +from metagpt.logs import logger class FileRepository: - def __init__(self, git_repo: GitRepository, relative_path: Path = "."): + def __init__(self, git_repo, relative_path: Path = Path(".")): self._relative_path = relative_path # Relative path based on the Git repository. self._git_repo = git_repo self._dependencies: Dict[str, List[str]] = {} - async def save(self, filename: Path, content, dependencies: List[str] = None): + # Initializing + self.workdir.mkdir(parents=True, exist_ok=True) + if self.dependency_path_name.exists(): + try: + with open(str(self.dependency_path_name), mode="r") as reader: + self._dependencies = json.load(reader) + except Exception as e: + logger.error(f"Failed to load {str(self.dependency_path_name)}, error:{e}") + + async def save(self, filename: Path | str, content, dependencies: List[str] = None): path_name = self.workdir / filename - with aiofiles.open(str(path_name), mode="w") as writer: + path_name.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(str(path_name), mode="w") as writer: await writer.write(content) if dependencies is not None: await self.update_dependency(filename, dependencies) + async def get(self, filename: Path | str): + path_name = self.workdir / filename + async with aiofiles.open(str(path_name), mode="r") as reader: + return await reader.read() + + def get_dependency(self, filename: Path | str) -> List: + key = str(filename) + return self._dependencies.get(key, []) + + def get_changed_dependency(self, filename: Path | str) -> List: + dependencies = self.get_dependency(filename=filename) + changed_files = self.changed_files + changed_dependent_files = [] + for df in dependencies: + if df in changed_files.keys(): + changed_dependent_files.append(df) + return changed_dependent_files + async def update_dependency(self, filename, dependencies: List[str]): self._dependencies[str(filename)] = dependencies async def save_dependency(self): - filename = ".dependencies.json" - path_name = self.workdir / filename data = json.dumps(self._dependencies) - with aiofiles.open(str(path_name), mode="w") as writer: + with aiofiles.open(str(self.dependency_path_name), mode="w") as writer: await writer.write(data) @property def workdir(self): return self._git_repo.workdir / self._relative_path + + @property + def dependency_path_name(self): + filename = ".dependencies.json" + path_name = self.workdir / filename + return path_name + + @property + def changed_files(self) -> Dict[str, str]: + files = self._git_repo.changed_files + relative_files = {} + for p, ct in files.items(): + try: + rf = Path(p).relative_to(self._relative_path) + except ValueError: + continue + relative_files[str(rf)] = ct + return relative_files + + def get_change_dir_files(self, dir: Path | str) -> List: + changed_files = self.changed_files + children = [] + for f in changed_files: + try: + Path(f).relative_to(Path(dir)) + except ValueError: + continue + children.append(str(f)) + return children diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 6e624c8b5..6ae6a7900 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -17,6 +17,7 @@ from git.repo.fun import is_git_dir from metagpt.const import WORKSPACE_ROOT +from metagpt.utils.file_repository import FileRepository class ChangeType(Enum): @@ -150,6 +151,14 @@ def archive(self, comments="Archive"): self.add_change(self.changed_files) self.commit(comments) + def new_file_repository(self, relative_path: Path | str) -> FileRepository: + """Create a new instance of FileRepository associated with this Git repository. + + :param relative_path: The relative path to the file repository within the Git repository. + :return: A new instance of FileRepository. + """ + return FileRepository(git_repo=self, relative_path=Path(relative_path)) + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py new file mode 100644 index 000000000..ac36f2320 --- /dev/null +++ b/tests/metagpt/utils/test_file_repository.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : test_file_repository.py +@Desc: Unit tests for file_repository.py +""" +import shutil +from pathlib import Path + +import pytest + +from metagpt.utils.git_repository import ChangeType, GitRepository +from tests.metagpt.utils.test_git_repository import mock_file + + +@pytest.mark.asyncio +async def test_file_repo(): + local_path = Path(__file__).parent / "file_repo_git" + if local_path.exists(): + shutil.rmtree(local_path) + + git_repo = GitRepository(local_path=local_path, auto_init=True) + assert not git_repo.changed_files + + await mock_file(local_path / "g.txt", "") + + file_repo_path = "file_repo1" + full_path = local_path / file_repo_path + assert not full_path.exists() + file_repo = git_repo.new_file_repository(file_repo_path) + assert file_repo.workdir == full_path + assert file_repo.workdir.exists() + await file_repo.save("a.txt", "AAA") + await file_repo.save("b.txt", "BBB", ["a.txt"]) + assert "AAA" == await file_repo.get("a.txt") + assert "BBB" == await file_repo.get("b.txt") + assert ["a.txt"] == file_repo.get_dependency("b.txt") + assert {"a.txt": ChangeType.UNTRACTED, "b.txt": ChangeType.UNTRACTED} == file_repo.changed_files + assert ["a.txt"] == file_repo.get_changed_dependency("b.txt") + await file_repo.save("d/e.txt", "EEE") + assert ["d/e.txt"] == file_repo.get_change_dir_files("d") + + git_repo.delete_repository() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 85e3620638348826f32f47065292c91e9e845193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 19:40:15 +0800 Subject: [PATCH 117/232] feat: +file repository --- metagpt/utils/file_repository.py | 51 +++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index d8637fe3f..f4c36b5b7 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -19,7 +19,12 @@ class FileRepository: def __init__(self, git_repo, relative_path: Path = Path(".")): - self._relative_path = relative_path # Relative path based on the Git repository. + """Initialize a FileRepository instance. + + :param git_repo: The associated GitRepository instance. + :param relative_path: The relative path within the Git repository. + """ + self._relative_path = relative_path self._git_repo = git_repo self._dependencies: Dict[str, List[str]] = {} @@ -33,6 +38,12 @@ def __init__(self, git_repo, relative_path: Path = Path(".")): logger.error(f"Failed to load {str(self.dependency_path_name)}, error:{e}") async def save(self, filename: Path | str, content, dependencies: List[str] = None): + """Save content to a file and update its dependencies. + + :param filename: The filename or path within the repository. + :param content: The content to be saved. + :param dependencies: List of dependency filenames or paths. + """ path_name = self.workdir / filename path_name.parent.mkdir(parents=True, exist_ok=True) async with aiofiles.open(str(path_name), mode="w") as writer: @@ -41,15 +52,30 @@ async def save(self, filename: Path | str, content, dependencies: List[str] = No await self.update_dependency(filename, dependencies) async def get(self, filename: Path | str): + """Read the content of a file. + + :param filename: The filename or path within the repository. + :return: The content of the file. + """ path_name = self.workdir / filename async with aiofiles.open(str(path_name), mode="r") as reader: return await reader.read() def get_dependency(self, filename: Path | str) -> List: + """Get the dependencies of a file. + + :param filename: The filename or path within the repository. + :return: List of dependency filenames or paths. + """ key = str(filename) return self._dependencies.get(key, []) def get_changed_dependency(self, filename: Path | str) -> List: + """Get the dependencies of a file that have changed. + + :param filename: The filename or path within the repository. + :return: List of changed dependency filenames or paths. + """ dependencies = self.get_dependency(filename=filename) changed_files = self.changed_files changed_dependent_files = [] @@ -59,25 +85,43 @@ def get_changed_dependency(self, filename: Path | str) -> List: return changed_dependent_files async def update_dependency(self, filename, dependencies: List[str]): + """Update the dependencies of a file. + + :param filename: The filename or path within the repository. + :param dependencies: List of dependency filenames or paths. + """ self._dependencies[str(filename)] = dependencies async def save_dependency(self): + """Save the dependencies to a file.""" data = json.dumps(self._dependencies) with aiofiles.open(str(self.dependency_path_name), mode="w") as writer: await writer.write(data) @property def workdir(self): + """Return the absolute path to the working directory of the FileRepository. + + :return: The absolute path to the working directory. + """ return self._git_repo.workdir / self._relative_path @property def dependency_path_name(self): + """Return the absolute path to the dependency file. + + :return: The absolute path to the dependency file. + """ filename = ".dependencies.json" path_name = self.workdir / filename return path_name @property def changed_files(self) -> Dict[str, str]: + """Return a dictionary of changed files and their change types. + + :return: A dictionary where keys are file paths and values are change types. + """ files = self._git_repo.changed_files relative_files = {} for p, ct in files.items(): @@ -89,6 +133,11 @@ def changed_files(self) -> Dict[str, str]: return relative_files def get_change_dir_files(self, dir: Path | str) -> List: + """Get the files in a directory that have changed. + + :param dir: The directory path within the repository. + :return: List of changed filenames or paths within the directory. + """ changed_files = self.changed_files children = [] for f in changed_files: From d9a2626fde3c7e43e7e118e3ee740a0ad4b9fcf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 21 Nov 2023 13:39:10 +0800 Subject: [PATCH 118/232] feat: +PrepareDocuments --- metagpt/actions/prepare_documents.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 metagpt/actions/prepare_documents.py diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py new file mode 100644 index 000000000..7cf05c5d1 --- /dev/null +++ b/metagpt/actions/prepare_documents.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. + RFC 135 2.2.3.5.1. +""" +from metagpt.actions import Action + + +class PrepareDocuments(Action): + def __init__(self, name="", context=None, llm=None): + pass From b73cbe73798647b7e1e69ff4c2c7f41ad9ec7c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 21 Nov 2023 14:00:09 +0800 Subject: [PATCH 119/232] feat: +unit test --- tests/metagpt/utils/test_common.py | 56 ++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index ec4443175..6474b1233 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -4,27 +4,79 @@ @Time : 2023/4/29 16:19 @Author : alexanderwu @File : test_common.py +@Modified by: mashenquan, 2023/11/21. Add unit tests. """ import os +from typing import Any, Set import pytest +from pydantic import BaseModel +from metagpt.actions import RunCode from metagpt.const import get_project_root +from metagpt.roles.tutorial_assistant import TutorialAssistant +from metagpt.schema import Message +from metagpt.utils.common import any_to_str, any_to_str_set class TestGetProjectRoot: def change_etc_dir(self): # current_directory = Path.cwd() - abs_root = '/etc' + abs_root = "/etc" os.chdir(abs_root) def test_get_project_root(self): project_root = get_project_root() - assert project_root.name == 'metagpt' + assert project_root.name == "MetaGPT" def test_get_root_exception(self): with pytest.raises(Exception) as exc_info: self.change_etc_dir() get_project_root() assert str(exc_info.value) == "Project root not found." + + def test_any_to_str(self): + class Input(BaseModel): + x: Any + want: str + + inputs = [ + Input(x=TutorialAssistant, want="metagpt.roles.tutorial_assistant.TutorialAssistant"), + Input(x=TutorialAssistant(), want="metagpt.roles.tutorial_assistant.TutorialAssistant"), + Input(x=RunCode, want="metagpt.actions.run_code.RunCode"), + Input(x=RunCode(), want="metagpt.actions.run_code.RunCode"), + Input(x=Message, want="metagpt.schema.Message"), + Input(x=Message(""), want="metagpt.schema.Message"), + Input(x="A", want="A"), + ] + for i in inputs: + v = any_to_str(i.x) + assert v == i.want + + def test_any_to_str_set(self): + class Input(BaseModel): + x: Any + want: Set + + inputs = [ + Input( + x=[TutorialAssistant, RunCode(), "a"], + want={"metagpt.roles.tutorial_assistant.TutorialAssistant", "metagpt.actions.run_code.RunCode", "a"}, + ), + Input( + x={TutorialAssistant, RunCode(), "a"}, + want={"metagpt.roles.tutorial_assistant.TutorialAssistant", "metagpt.actions.run_code.RunCode", "a"}, + ), + Input( + x=(TutorialAssistant, RunCode(), "a"), + want={"metagpt.roles.tutorial_assistant.TutorialAssistant", "metagpt.actions.run_code.RunCode", "a"}, + ), + ] + for i in inputs: + v = any_to_str_set(i.x) + assert v == i.want + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 27c731d11a94901d4b51dbfe57042ee1a9681b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 21 Nov 2023 15:05:23 +0800 Subject: [PATCH 120/232] feat: archive --- metagpt/actions/prepare_documents.py | 12 +++++++++++- metagpt/environment.py | 9 ++++++++- metagpt/roles/product_manager.py | 11 ++++++++++- metagpt/roles/role.py | 4 ++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 7cf05c5d1..b0185996b 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -12,4 +12,14 @@ class PrepareDocuments(Action): def __init__(self, name="", context=None, llm=None): - pass + super().__init__(name, context, llm) + + async def run(self, with_message, **kwargs): + parent = self.context.get("parent") + if not parent: + raise ValueError("Invalid owner") + env = parent.get_env() + if env.git_repository: + return + env.git_repository = GitRepository() + env.git_repository.open(WORKS) diff --git a/metagpt/environment.py b/metagpt/environment.py index b3c296dac..df93a818b 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -12,7 +12,7 @@ functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable, Set +from typing import Iterable, Optional, Set from pydantic import BaseModel, Field @@ -20,6 +20,7 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import is_subscribed +from metagpt.utils.git_repository import GitRepository class Environment(BaseModel): @@ -31,6 +32,7 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) consumers: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug + git_repository: Optional[GitRepository] = None class Config: arbitrary_types_allowed = True @@ -111,3 +113,8 @@ def get_subscription(self, obj): def set_subscription(self, obj, tags): """Set the labels for message to be consumed by the object""" self.consumers[obj] = tags + + def dict(self, *args, **kwargs): + """Generate a dictionary representation of the model, optionally specifying which fields to include or + exclude.""" + return super(Environment, self).dict(exclude={"git_repository"}) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..c10aba6d1 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -6,6 +6,7 @@ @File : product_manager.py """ from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles import Role @@ -37,5 +38,13 @@ def __init__( constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([WritePRD]) + self._init_actions([PrepareDocuments(context={"parent": self}), WritePRD]) self._watch([BossRequirement]) + + async def _think(self) -> None: + """Decide what to do""" + if self._rc.env.git_repository: + self._set_state(1) + else: + self._set_state(0) + return self._rc.todo diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 2e3bcbbd5..d1e65a4e0 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -164,6 +164,10 @@ def set_env(self, env: "Environment"): if env: env.set_subscription(self, self._subscription) + def get_env(self): + """Return the environment in which the role works.""" + return self._rc.env + @property def profile(self): """Get the role description (position)""" From 5142cb59f7120e564d014e8e3ab2e69698e0972e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 15:59:35 +0800 Subject: [PATCH 121/232] refactor: consumers -> members --- metagpt/environment.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/environment.py b/metagpt/environment.py index b3c296dac..02eb3d340 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -29,7 +29,7 @@ class Environment(BaseModel): """ roles: dict[str, Role] = Field(default_factory=dict) - consumers: dict[Role, Set] = Field(default_factory=dict) + members: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug class Config: @@ -61,7 +61,7 @@ def publish_message(self, message: Message) -> bool: logger.info(f"publish_message: {message.dump()}") found = False # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 - for role, subscription in self.consumers.items(): + for role, subscription in self.members.items(): if is_subscribed(message, subscription): role.put_message(message) found = True @@ -106,8 +106,8 @@ def is_idle(self): def get_subscription(self, obj): """Get the labels for messages to be consumed by the object.""" - return self.consumers.get(obj, {}) + return self.members.get(obj, {}) def set_subscription(self, obj, tags): """Set the labels for message to be consumed by the object""" - self.consumers[obj] = tags + self.members[obj] = tags From cda032948f855995d7d4a21fef5b0dc800a47499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 16:26:48 +0800 Subject: [PATCH 122/232] refactor: pre-commit run --all-files --- examples/agent_creator.py | 16 +-- examples/build_customized_agent.py | 22 ++-- examples/debate.py | 27 +++-- examples/invoice_ocr.py | 10 +- examples/llm_hello_world.py | 10 +- examples/research.py | 2 +- examples/search_google.py | 2 +- examples/search_kb.py | 4 +- examples/search_with_specific_engine.py | 7 +- examples/use_off_the_shelf_agent.py | 10 +- examples/write_tutorial.py | 3 +- metagpt/actions/action_output.py | 7 +- metagpt/actions/add_requirement.py | 1 + metagpt/actions/azure_tts.py | 18 +-- metagpt/actions/clone_function.py | 6 +- metagpt/actions/debug_error.py | 10 +- metagpt/actions/design_api_review.py | 9 +- metagpt/actions/design_filenames.py | 7 +- metagpt/actions/detail_mining.py | 5 +- metagpt/actions/invoice_ocr.py | 8 +- metagpt/actions/prepare_interview.py | 1 - metagpt/actions/research.py | 27 +++-- metagpt/actions/search_and_summarize.py | 1 - metagpt/actions/write_code.py | 6 +- metagpt/actions/write_code_review.py | 6 +- metagpt/actions/write_docstring.py | 7 +- metagpt/actions/write_prd_review.py | 1 - metagpt/actions/write_tutorial.py | 3 +- metagpt/config.py | 2 +- metagpt/document_store/base_store.py | 3 +- metagpt/document_store/chromadb_store.py | 3 +- metagpt/document_store/document.py | 20 ++-- metagpt/document_store/faiss_store.py | 14 +-- metagpt/document_store/milvus_store.py | 23 +--- metagpt/document_store/qdrant_store.py | 27 ++--- metagpt/environment.py | 16 +-- metagpt/inspect_module.py | 10 +- metagpt/llm.py | 5 +- metagpt/logs.py | 6 +- metagpt/management/skill_manager.py | 8 +- metagpt/manager.py | 4 +- metagpt/memory/longterm_memory.py | 1 - metagpt/memory/memory.py | 1 - metagpt/memory/memory_storage.py | 24 ++-- metagpt/prompts/invoice_ocr.py | 11 +- metagpt/prompts/sales.py | 16 +-- metagpt/prompts/tutorial_assistant.py | 12 +- metagpt/provider/anthropic_api.py | 1 - metagpt/provider/base_chatbot.py | 2 +- metagpt/provider/base_gpt_api.py | 6 +- metagpt/provider/openai_api.py | 2 - metagpt/provider/spark_api.py | 61 ++++------- metagpt/roles/customer_service.py | 9 +- metagpt/roles/invoice_ocr_assistant.py | 3 +- metagpt/roles/prompt.py | 5 +- metagpt/roles/role.py | 34 ++++-- metagpt/roles/sales.py | 21 ++-- metagpt/roles/seacher.py | 32 +++--- metagpt/roles/tutorial_assistant.py | 4 +- metagpt/schema.py | 29 ++--- metagpt/software_company.py | 6 +- metagpt/tools/code_interpreter.py | 61 ++++++----- metagpt/tools/prompt_writer.py | 7 +- metagpt/tools/sd_engine.py | 2 + metagpt/tools/search_engine.py | 6 +- metagpt/tools/search_engine_meilisearch.py | 4 +- metagpt/tools/translator.py | 9 +- metagpt/tools/ut_writer.py | 28 +++-- metagpt/utils/file.py | 6 +- metagpt/utils/highlight.py | 12 +- metagpt/utils/mmdc_ink.py | 6 +- metagpt/utils/mmdc_playwright.py | 94 +++++++++------- metagpt/utils/mmdc_pyppeteer.py | 103 ++++++++++-------- metagpt/utils/parse_html.py | 4 +- metagpt/utils/pycst.py | 12 +- metagpt/utils/read_document.py | 1 + metagpt/utils/singleton.py | 1 - metagpt/utils/special_tokens.py | 2 +- metagpt/utils/text.py | 13 ++- tests/conftest.py | 4 +- tests/metagpt/actions/mock.py | 14 +-- tests/metagpt/actions/test_action_output.py | 29 ++--- tests/metagpt/actions/test_azure_tts.py | 7 +- tests/metagpt/actions/test_clone_function.py | 21 ++-- tests/metagpt/actions/test_debug_error.py | 6 +- tests/metagpt/actions/test_detail_mining.py | 6 +- tests/metagpt/actions/test_invoice_ocr.py | 26 +---- tests/metagpt/actions/test_ui_design.py | 20 ++-- tests/metagpt/actions/test_write_code.py | 6 +- tests/metagpt/actions/test_write_docstring.py | 6 +- tests/metagpt/actions/test_write_tutorial.py | 9 +- .../document_store/test_chromadb_store.py | 8 +- .../document_store/test_faiss_store.py | 13 ++- .../document_store/test_lancedb_store.py | 28 +++-- .../document_store/test_milvus_store.py | 10 +- .../document_store/test_qdrant_store.py | 12 +- .../metagpt/management/test_skill_manager.py | 2 +- tests/metagpt/memory/test_longterm_memory.py | 24 ++-- tests/metagpt/memory/test_memory_storage.py | 62 ++++------- tests/metagpt/provider/test_base_gpt_api.py | 6 +- tests/metagpt/provider/test_spark_api.py | 2 +- tests/metagpt/roles/mock.py | 18 +-- tests/metagpt/roles/test_engineer.py | 4 +- .../roles/test_invoice_ocr_assistant.py | 67 ++---------- tests/metagpt/roles/test_researcher.py | 6 +- .../metagpt/roles/test_tutorial_assistant.py | 7 +- tests/metagpt/roles/test_ui.py | 3 +- tests/metagpt/test_gpt.py | 10 +- tests/metagpt/test_llm.py | 6 +- tests/metagpt/test_message.py | 18 +-- tests/metagpt/test_role.py | 6 +- tests/metagpt/test_schema.py | 6 +- tests/metagpt/tools/test_code_interpreter.py | 29 ++--- tests/metagpt/tools/test_prompt_generator.py | 12 +- tests/metagpt/tools/test_sd_tool.py | 2 +- tests/metagpt/tools/test_search_engine.py | 12 +- .../tools/test_search_engine_meilisearch.py | 6 +- tests/metagpt/tools/test_summarize.py | 3 +- tests/metagpt/tools/test_translate.py | 2 +- tests/metagpt/tools/test_ut_generator.py | 8 +- tests/metagpt/utils/test_code_parser.py | 4 +- tests/metagpt/utils/test_common.py | 4 +- tests/metagpt/utils/test_config.py | 6 +- .../metagpt/utils/test_custom_aio_session.py | 4 +- tests/metagpt/utils/test_file.py | 5 +- tests/metagpt/utils/test_output_parser.py | 24 ++-- tests/metagpt/utils/test_parse_html.py | 8 +- tests/metagpt/utils/test_pycst.py | 4 +- tests/metagpt/utils/test_text.py | 8 +- 129 files changed, 811 insertions(+), 830 deletions(-) diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 325e7c260..3618c0608 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -1,22 +1,22 @@ -''' +""" Filename: MetaGPT/examples/agent_creator.py Created Date: Tuesday, September 12th 2023, 3:28:37 pm Author: garylin2099 -''' +""" import re -from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT from metagpt.actions import Action +from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() -class CreateAgent(Action): +class CreateAgent(Action): PROMPT_TEMPLATE = """ ### BACKGROUND You are using an agent framework called metagpt to write agents capable of different actions, @@ -34,7 +34,6 @@ class CreateAgent(Action): """ async def run(self, example: str, instruction: str): - prompt = self.PROMPT_TEMPLATE.format(example=example, instruction=instruction) # logger.info(prompt) @@ -46,13 +45,14 @@ async def run(self, example: str, instruction: str): @staticmethod def parse_code(rsp): - pattern = r'```python(.*)```' + pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text + class AgentCreator(Role): def __init__( self, @@ -76,11 +76,11 @@ async def _act(self) -> Message: return msg + if __name__ == "__main__": import asyncio async def main(): - agent_template = MULTI_ACTION_AGENT_CODE_EXAMPLE creator = AgentCreator(agent_template=agent_template) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index 87d7a9c76..ef274be8b 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -1,21 +1,21 @@ -''' +""" Filename: MetaGPT/examples/build_customized_agent.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +""" +import asyncio import re import subprocess -import asyncio import fire from metagpt.actions import Action +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger -class SimpleWriteCode(Action): +class SimpleWriteCode(Action): PROMPT_TEMPLATE = """ Write a python function that can {instruction} and provide two runnnable test cases. Return ```python your_code_here ``` with NO other texts, @@ -35,7 +35,6 @@ def __init__(self, name="SimpleWriteCode", context=None, llm=None): super().__init__(name, context, llm) async def run(self, instruction: str): - prompt = self.PROMPT_TEMPLATE.format(instruction=instruction) rsp = await self._aask(prompt) @@ -46,11 +45,12 @@ async def run(self, instruction: str): @staticmethod def parse_code(rsp): - pattern = r'```python(.*)```' + pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else rsp return code_text + class SimpleRunCode(Action): def __init__(self, name="SimpleRunCode", context=None, llm=None): super().__init__(name, context, llm) @@ -61,6 +61,7 @@ async def run(self, code_text: str): logger.info(f"{code_result=}") return code_result + class SimpleCoder(Role): def __init__( self, @@ -75,7 +76,7 @@ async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo - msg = self._rc.memory.get()[-1] # retrieve the latest memory + msg = self._rc.memory.get()[-1] # retrieve the latest memory instruction = msg.content code_text = await SimpleWriteCode().run(instruction) @@ -83,6 +84,7 @@ async def _act(self) -> Message: return msg + class RunnableCoder(Role): def __init__( self, @@ -128,6 +130,7 @@ async def _react(self) -> Message: await self._act() return Message(content="All job done", role=self.profile) + def main(msg="write a function that calculates the sum of a list"): # role = SimpleCoder() role = RunnableCoder() @@ -135,5 +138,6 @@ def main(msg="write a function that calculates the sum of a list"): result = asyncio.run(role.run(msg)) logger.info(result) -if __name__ == '__main__': + +if __name__ == "__main__": fire.Fire(main) diff --git a/examples/debate.py b/examples/debate.py index 05db28070..54da73cca 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -1,17 +1,19 @@ -''' +""" Filename: MetaGPT/examples/debate.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +""" import asyncio import platform + import fire -from metagpt.software_company import SoftwareCompany from metagpt.actions import Action, BossRequirement +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger +from metagpt.software_company import SoftwareCompany + class ShoutOut(Action): """Action: Shout out loudly in a debate (quarrel)""" @@ -31,7 +33,6 @@ def __init__(self, name="ShoutOut", context=None, llm=None): super().__init__(name, context, llm) async def run(self, context: str, name: str, opponent_name: str): - prompt = self.PROMPT_TEMPLATE.format(context=context, name=name, opponent_name=opponent_name) # logger.info(prompt) @@ -39,6 +40,7 @@ async def run(self, context: str, name: str, opponent_name: str): return rsp + class Trump(Role): def __init__( self, @@ -55,7 +57,7 @@ def __init__( async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] + self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] return len(self._rc.news) async def _act(self) -> Message: @@ -79,6 +81,7 @@ async def _act(self) -> Message: return msg + class Biden(Role): def __init__( self, @@ -120,10 +123,12 @@ async def _act(self) -> Message: return msg -async def startup(idea: str, investment: float = 3.0, n_round: int = 5, - code_review: bool = False, run_tests: bool = False): + +async def startup( + idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False +): """We reuse the startup paradigm for roles to interact with each other. - Now we run a startup of presidents and watch they quarrel. :) """ + Now we run a startup of presidents and watch they quarrel. :)""" company = SoftwareCompany() company.hire([Biden(), Trump()]) company.invest(investment) @@ -133,7 +138,7 @@ async def startup(idea: str, investment: float = 3.0, n_round: int = 5, def main(idea: str, investment: float = 3.0, n_round: int = 10): """ - :param idea: Debate topic, such as "Topic: The U.S. should commit more in climate change fighting" + :param idea: Debate topic, such as "Topic: The U.S. should commit more in climate change fighting" or "Trump: Climate change is a hoax" :param investment: contribute a certain dollar amount to watch the debate :param n_round: maximum rounds of the debate @@ -144,5 +149,5 @@ def main(idea: str, investment: float = 3.0, n_round: int = 10): asyncio.run(startup(idea, investment, n_round)) -if __name__ == '__main__': +if __name__ == "__main__": fire.Fire(main) diff --git a/examples/invoice_ocr.py b/examples/invoice_ocr.py index 11656ed52..a6e565772 100644 --- a/examples/invoice_ocr.py +++ b/examples/invoice_ocr.py @@ -19,19 +19,15 @@ async def main(): Path("../tests/data/invoices/invoice-1.pdf"), Path("../tests/data/invoices/invoice-2.png"), Path("../tests/data/invoices/invoice-3.jpg"), - Path("../tests/data/invoices/invoice-4.zip") + Path("../tests/data/invoices/invoice-4.zip"), ] # The absolute path of the file absolute_file_paths = [Path.cwd() / path for path in relative_paths] for path in absolute_file_paths: role = InvoiceOCRAssistant() - await role.run(Message( - content="Invoicing date", - instruct_content={"file_path": path} - )) + await role.run(Message(content="Invoicing date", instruct_content={"file_path": path})) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) - diff --git a/examples/llm_hello_world.py b/examples/llm_hello_world.py index 3ba03eea0..677098399 100644 --- a/examples/llm_hello_world.py +++ b/examples/llm_hello_world.py @@ -14,11 +14,11 @@ async def main(): llm = LLM() claude = Claude() - logger.info(await claude.aask('你好，请进行自我介绍')) - logger.info(await llm.aask('hello world')) - logger.info(await llm.aask_batch(['hi', 'write python hello world.'])) + logger.info(await claude.aask("你好，请进行自我介绍")) + logger.info(await llm.aask("hello world")) + logger.info(await llm.aask_batch(["hi", "write python hello world."])) - hello_msg = [{'role': 'user', 'content': 'count from 1 to 10. split by newline.'}] + hello_msg = [{"role": "user", "content": "count from 1 to 10. split by newline."}] logger.info(await llm.acompletion(hello_msg)) logger.info(await llm.acompletion_batch([hello_msg])) logger.info(await llm.acompletion_batch_text([hello_msg])) @@ -27,5 +27,5 @@ async def main(): await llm.acompletion_text(hello_msg, stream=True) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/research.py b/examples/research.py index 344f8d0e9..5c371cdd2 100644 --- a/examples/research.py +++ b/examples/research.py @@ -12,5 +12,5 @@ async def main(): print(f"save report to {RESEARCH_PATH / f'{topic}.md'}.") -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/search_google.py b/examples/search_google.py index 9e9521b9c..73d04bf87 100644 --- a/examples/search_google.py +++ b/examples/search_google.py @@ -15,5 +15,5 @@ async def main(): await Searcher().run("What are some good sun protection products?") -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/search_kb.py b/examples/search_kb.py index b6f7d87a0..0b5d59385 100644 --- a/examples/search_kb.py +++ b/examples/search_kb.py @@ -12,7 +12,7 @@ async def search(): - store = FaissStore(DATA_PATH / 'example.json') + store = FaissStore(DATA_PATH / "example.json") role = Sales(profile="Sales", store=store) queries = ["Which facial cleanser is good for oily skin?", "Is L'Oreal good to use?"] @@ -22,5 +22,5 @@ async def search(): logger.info(result) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(search()) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 7cc431cd4..334a7821f 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -6,11 +6,12 @@ async def main(): # Serper API - #await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run(["What are some good sun protection products?","What are some of the best beaches?"]) + # await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run(["What are some good sun protection products?","What are some of the best beaches?"]) # SerpAPI - #await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run("What are the best ski brands for skiers?") + # await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run("What are the best ski brands for skiers?") # Google API await Searcher(engine=SearchEngineType.DIRECT_GOOGLE).run("What are the most interesting human facts?") -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/use_off_the_shelf_agent.py b/examples/use_off_the_shelf_agent.py index 2e10068bd..4445a6c62 100644 --- a/examples/use_off_the_shelf_agent.py +++ b/examples/use_off_the_shelf_agent.py @@ -1,12 +1,13 @@ -''' +""" Filename: MetaGPT/examples/use_off_the_shelf_agent.py Created Date: Tuesday, September 19th 2023, 6:52:25 pm Author: garylin2099 -''' +""" import asyncio -from metagpt.roles.product_manager import ProductManager from metagpt.logs import logger +from metagpt.roles.product_manager import ProductManager + async def main(): msg = "Write a PRD for a snake game" @@ -14,5 +15,6 @@ async def main(): result = await role.run(msg) logger.info(result.content[:100]) -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/examples/write_tutorial.py b/examples/write_tutorial.py index 71ece5527..0dba3cdb7 100644 --- a/examples/write_tutorial.py +++ b/examples/write_tutorial.py @@ -16,6 +16,5 @@ async def main(): await role.run(topic) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) - diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index ea7f4fb80..25326d43b 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -23,10 +23,10 @@ def __init__(self, content: str, instruct_content: BaseModel): def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): new_class = create_model(class_name, **mapping) - @validator('*', allow_reuse=True) + @validator("*", allow_reuse=True) def check_name(v, field): if field.name not in mapping.keys(): - raise ValueError(f'Unrecognized block: {field.name}') + raise ValueError(f"Unrecognized block: {field.name}") return v @root_validator(pre=True, allow_reuse=True) @@ -34,10 +34,9 @@ def check_missing_fields(values): required_fields = set(mapping.keys()) missing_fields = required_fields - set(values.keys()) if missing_fields: - raise ValueError(f'Missing fields: {missing_fields}') + raise ValueError(f"Missing fields: {missing_fields}") return values new_class.__validator_check_name = classmethod(check_name) new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields) return new_class - \ No newline at end of file diff --git a/metagpt/actions/add_requirement.py b/metagpt/actions/add_requirement.py index 7dc09d062..16e14b3a4 100644 --- a/metagpt/actions/add_requirement.py +++ b/metagpt/actions/add_requirement.py @@ -10,5 +10,6 @@ class BossRequirement(Action): """Boss Requirement without any implementation details""" + async def run(self, *args, **kwargs): raise NotImplementedError diff --git a/metagpt/actions/azure_tts.py b/metagpt/actions/azure_tts.py index c13a4750d..daa3f6892 100644 --- a/metagpt/actions/azure_tts.py +++ b/metagpt/actions/azure_tts.py @@ -18,16 +18,13 @@ def __init__(self, name, context=None, llm=None): # Parameters reference: https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles def synthesize_speech(self, lang, voice, role, text, output_file): - subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY') - region = self.config.get('AZURE_TTS_REGION') - speech_config = SpeechConfig( - subscription=subscription_key, region=region) + subscription_key = self.config.get("AZURE_TTS_SUBSCRIPTION_KEY") + region = self.config.get("AZURE_TTS_REGION") + speech_config = SpeechConfig(subscription=subscription_key, region=region) speech_config.speech_synthesis_voice_name = voice audio_config = AudioConfig(filename=output_file) - synthesizer = SpeechSynthesizer( - speech_config=speech_config, - audio_config=audio_config) + synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=audio_config) # if voice=="zh-CN-YunxiNeural": ssml_string = f""" @@ -45,9 +42,4 @@ def synthesize_speech(self, lang, voice, role, text, output_file): if __name__ == "__main__": azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( - "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "Hello, I am Kaka", - "output.wav") + azure_tts.synthesize_speech("zh-CN", "zh-CN-YunxiNeural", "Boy", "Hello, I am Kaka", "output.wav") diff --git a/metagpt/actions/clone_function.py b/metagpt/actions/clone_function.py index cf7d22f04..1447e8dbf 100644 --- a/metagpt/actions/clone_function.py +++ b/metagpt/actions/clone_function.py @@ -1,5 +1,5 @@ -from pathlib import Path import traceback +from pathlib import Path from metagpt.actions.write_code import WriteCode from metagpt.logs import logger @@ -42,7 +42,7 @@ async def run(self, template_func: str, source_code: str) -> str: prompt = CLONE_PROMPT.format(source_code=source_code, template_func=template_func) logger.info(f"query for CloneFunction: \n {prompt}") code = await self.write_code(prompt) - logger.info(f'CloneFunction code is \n {highlight(code)}') + logger.info(f"CloneFunction code is \n {highlight(code)}") return code @@ -61,5 +61,5 @@ def run_function_script(code_script_path: str, func_name: str, *args, **kwargs): """Run function code from script.""" if isinstance(code_script_path, str): code_path = Path(code_script_path) - code = code_path.read_text(encoding='utf-8') + code = code_path.read_text(encoding="utf-8") return run_function_code(code, func_name, *args, **kwargs) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index d69a22dba..304b1bc3e 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -7,8 +7,8 @@ """ import re -from metagpt.logs import logger from metagpt.actions.action import Action +from metagpt.logs import logger from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -24,6 +24,8 @@ Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. """ + + class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) @@ -33,17 +35,17 @@ def __init__(self, name="DebugError", context=None, llm=None): # f"\n\n{error}\n\nPlease try to fix the error in this code." # fixed_code = await self._aask(prompt) # return fixed_code - + async def run(self, context): if "PASS" in context: return "", "the original code works fine, no need to debug" - + file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) logger.info(f"Debug and rewrite {file_name}") prompt = PROMPT_TEMPLATE.format(context=context) - + rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=rsp) diff --git a/metagpt/actions/design_api_review.py b/metagpt/actions/design_api_review.py index 9bb822a62..7f25bb9a3 100644 --- a/metagpt/actions/design_api_review.py +++ b/metagpt/actions/design_api_review.py @@ -13,10 +13,11 @@ def __init__(self, name, context=None, llm=None): super().__init__(name, context, llm) async def run(self, prd, api_design): - prompt = f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " \ - f"based on this PRD:\n\n{api_design}\n\nPlease review whether this API design meets the requirements" \ - f" of the PRD, and whether it complies with good design practices." + prompt = ( + f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " + f"based on this PRD:\n\n{api_design}\n\nPlease review whether this API design meets the requirements" + f" of the PRD, and whether it complies with good design practices." + ) api_review = await self._aask(prompt) return api_review - \ No newline at end of file diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py index 29400e950..ffa171d7b 100644 --- a/metagpt/actions/design_filenames.py +++ b/metagpt/actions/design_filenames.py @@ -17,8 +17,10 @@ class DesignFilenames(Action): def __init__(self, name, context=None, llm=None): super().__init__(name, context, llm) - self.desc = "Based on the PRD, consider system design, and carry out the basic design of the corresponding " \ - "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." + self.desc = ( + "Based on the PRD, consider system design, and carry out the basic design of the corresponding " + "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." + ) async def run(self, prd): prompt = f"The following is the Product Requirement Document (PRD):\n\n{prd}\n\n{PROMPT}" @@ -26,4 +28,3 @@ async def run(self, prd): logger.debug(prompt) logger.debug(design_filenames) return design_filenames - \ No newline at end of file diff --git a/metagpt/actions/detail_mining.py b/metagpt/actions/detail_mining.py index e29d6911b..5afcf52c6 100644 --- a/metagpt/actions/detail_mining.py +++ b/metagpt/actions/detail_mining.py @@ -6,7 +6,6 @@ @File : detail_mining.py """ from metagpt.actions import Action, ActionOutput -from metagpt.logs import logger PROMPT_TEMPLATE = """ ##TOPIC @@ -41,8 +40,8 @@ class DetailMining(Action): - """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and "##RECORD" (discussion records), thereby deepening the discussion. - """ + """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and "##RECORD" (discussion records), thereby deepening the discussion.""" + def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) diff --git a/metagpt/actions/invoice_ocr.py b/metagpt/actions/invoice_ocr.py index b37aa6885..dcf537a58 100644 --- a/metagpt/actions/invoice_ocr.py +++ b/metagpt/actions/invoice_ocr.py @@ -10,8 +10,8 @@ import os import zipfile -from pathlib import Path from datetime import datetime +from pathlib import Path import pandas as pd from paddleocr import PaddleOCR @@ -19,7 +19,10 @@ from metagpt.actions import Action from metagpt.const import INVOICE_OCR_TABLE_PATH from metagpt.logs import logger -from metagpt.prompts.invoice_ocr import EXTRACT_OCR_MAIN_INFO_PROMPT, REPLY_OCR_QUESTION_PROMPT +from metagpt.prompts.invoice_ocr import ( + EXTRACT_OCR_MAIN_INFO_PROMPT, + REPLY_OCR_QUESTION_PROMPT, +) from metagpt.utils.common import OutputParser from metagpt.utils.file import File @@ -183,4 +186,3 @@ async def run(self, query: str, ocr_result: list, *args, **kwargs) -> str: prompt = REPLY_OCR_QUESTION_PROMPT.format(query=query, ocr_result=ocr_result, language=self.language) resp = await self._aask(prompt=prompt) return resp - diff --git a/metagpt/actions/prepare_interview.py b/metagpt/actions/prepare_interview.py index 5db3a9f37..b2704616e 100644 --- a/metagpt/actions/prepare_interview.py +++ b/metagpt/actions/prepare_interview.py @@ -38,4 +38,3 @@ async def run(self, context): prompt = PROMPT_TEMPLATE.format(context=context) question_list = await self._aask_v1(prompt) return question_list - diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 49a981e86..d7a2a7e38 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import json from typing import Callable from pydantic import parse_obj_as @@ -49,7 +48,7 @@ ranked results' indices in JSON format, like [0, 1, 3, 4, ...], without including other words. """ -WEB_BROWSE_AND_SUMMARIZE_PROMPT = '''### Requirements +WEB_BROWSE_AND_SUMMARIZE_PROMPT = """### Requirements 1. Utilize the text in the "Reference Information" section to respond to the question "{query}". 2. If the question cannot be directly answered using the text, but the text is related to the research topic, please provide \ a comprehensive summary of the text. @@ -58,10 +57,10 @@ ### Reference Information {content} -''' +""" -CONDUCT_RESEARCH_PROMPT = '''### Reference Information +CONDUCT_RESEARCH_PROMPT = """### Reference Information {content} ### Requirements @@ -73,11 +72,12 @@ - Present data and findings in an intuitive manner, utilizing feature comparative tables, if applicable. - The report should have a minimum word count of 2,000 and be formatted with Markdown syntax following APA style guidelines. - Include all source URLs in APA format at the end of the report. -''' +""" class CollectLinks(Action): """Action class to collect links from a search engine.""" + def __init__( self, name: str = "", @@ -114,19 +114,24 @@ async def run( keywords = OutputParser.extract_struct(keywords, list) keywords = parse_obj_as(list[str], keywords) except Exception as e: - logger.exception(f"fail to get keywords related to the research topic \"{topic}\" for {e}") + logger.exception(f'fail to get keywords related to the research topic "{topic}" for {e}') keywords = [topic] results = await asyncio.gather(*(self.search_engine.run(i, as_string=False) for i in keywords)) def gen_msg(): while True: - search_results = "\n".join(f"#### Keyword: {i}\n Search Result: {j}\n" for (i, j) in zip(keywords, results)) - prompt = SUMMARIZE_SEARCH_PROMPT.format(decomposition_nums=decomposition_nums, search_results=search_results) + search_results = "\n".join( + f"#### Keyword: {i}\n Search Result: {j}\n" for (i, j) in zip(keywords, results) + ) + prompt = SUMMARIZE_SEARCH_PROMPT.format( + decomposition_nums=decomposition_nums, search_results=search_results + ) yield prompt remove = max(results, key=len) remove.pop() if len(remove) == 0: break + prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, CONFIG.max_tokens_rsp) logger.debug(prompt) queries = await self._aask(prompt, [system_text]) @@ -172,6 +177,7 @@ async def _search_and_rank_urls(self, topic: str, query: str, num_results: int = class WebBrowseAndSummarize(Action): """Action class to explore the web and provide summaries of articles and webpages.""" + def __init__( self, *args, @@ -214,7 +220,9 @@ async def run( for u, content in zip([url, *urls], contents): content = content.inner_text chunk_summaries = [] - for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp): + for prompt in generate_prompt_chunk( + content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp + ): logger.debug(prompt) summary = await self._aask(prompt, [system_text]) if summary == "Not relevant.": @@ -238,6 +246,7 @@ async def run( class ConductResearch(Action): """Action class to conduct research and generate a research report.""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if CONFIG.model_for_researcher_report: diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 069f2a977..5e4cdaea0 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -140,4 +140,3 @@ async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYS logger.debug(prompt) logger.debug(result) return result - \ No newline at end of file diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c000805c5..a922d3694 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,13 +5,14 @@ @Author : alexanderwu @File : write_code.py """ +from tenacity import retry, stop_after_attempt, wait_fixed + from metagpt.actions import WriteDesign from metagpt.actions.action import Action from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -74,9 +75,8 @@ async def write_code(self, prompt): async def run(self, context, filename): prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f'Writing {filename}..') + logger.info(f"Writing {filename}..") code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) return code - \ No newline at end of file diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 4ff4d6cf6..76adca255 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -6,11 +6,12 @@ @File : write_code_review.py """ +from tenacity import retry, stop_after_attempt, wait_fixed + from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -74,9 +75,8 @@ async def write_code(self, prompt): async def run(self, context, code, filename): format_example = FORMAT_EXAMPLE.format(filename=filename) prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) - logger.info(f'Code review {filename}..') + logger.info(f"Code review {filename}..") code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) return code - \ No newline at end of file diff --git a/metagpt/actions/write_docstring.py b/metagpt/actions/write_docstring.py index 5c7815793..dd3312bd5 100644 --- a/metagpt/actions/write_docstring.py +++ b/metagpt/actions/write_docstring.py @@ -28,7 +28,7 @@ from metagpt.utils.common import OutputParser from metagpt.utils.pycst import merge_docstring -PYTHON_DOCSTRING_SYSTEM = '''### Requirements +PYTHON_DOCSTRING_SYSTEM = """### Requirements 1. Add docstrings to the given code following the {style} style. 2. Replace the function body with an Ellipsis object(...) to reduce output. 3. If the types are already annotated, there is no need to include them in the docstring. @@ -48,7 +48,7 @@ def __init__(self, msg: str): ```python {example} ``` -''' +""" # https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html @@ -162,7 +162,8 @@ def __init__(self, *args, **kwargs): self.desc = "Write docstring for code." async def run( - self, code: str, + self, + code: str, system_text: str = PYTHON_DOCSTRING_SYSTEM, style: Literal["google", "numpy", "sphinx"] = "google", ) -> str: diff --git a/metagpt/actions/write_prd_review.py b/metagpt/actions/write_prd_review.py index 5c922d3bc..5ff9624c5 100644 --- a/metagpt/actions/write_prd_review.py +++ b/metagpt/actions/write_prd_review.py @@ -25,4 +25,3 @@ async def run(self, prd): prompt = self.prd_review_prompt_template.format(prd=self.prd) review = await self._aask(prompt) return review - \ No newline at end of file diff --git a/metagpt/actions/write_tutorial.py b/metagpt/actions/write_tutorial.py index 23e3560e8..d41915de3 100644 --- a/metagpt/actions/write_tutorial.py +++ b/metagpt/actions/write_tutorial.py @@ -10,7 +10,7 @@ from typing import Dict from metagpt.actions import Action -from metagpt.prompts.tutorial_assistant import DIRECTORY_PROMPT, CONTENT_PROMPT +from metagpt.prompts.tutorial_assistant import CONTENT_PROMPT, DIRECTORY_PROMPT from metagpt.utils.common import OutputParser @@ -65,4 +65,3 @@ async def run(self, topic: str, *args, **kwargs) -> str: """ prompt = CONTENT_PROMPT.format(topic=topic, language=self.language, directory=self.directory) return await self._aask(prompt=prompt) - diff --git a/metagpt/config.py b/metagpt/config.py index 27455d38d..d93640c1b 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -46,7 +46,7 @@ def __init__(self, yaml_file=default_yaml_file): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") diff --git a/metagpt/document_store/base_store.py b/metagpt/document_store/base_store.py index 5d7015e8b..7d102e00b 100644 --- a/metagpt/document_store/base_store.py +++ b/metagpt/document_store/base_store.py @@ -41,7 +41,7 @@ def __init__(self, raw_data: Path, cache_dir: Path = None): self.store = self.write() def _get_index_and_store_fname(self): - fname = self.raw_data.name.split('.')[0] + fname = self.raw_data.name.split(".")[0] index_file = self.cache_dir / f"{fname}.index" store_file = self.cache_dir / f"{fname}.pkl" return index_file, store_file @@ -53,4 +53,3 @@ def _load(self): @abstractmethod def _write(self, docs, metadatas): raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/chromadb_store.py b/metagpt/document_store/chromadb_store.py index d2ecc05f6..d7344d41b 100644 --- a/metagpt/document_store/chromadb_store.py +++ b/metagpt/document_store/chromadb_store.py @@ -10,6 +10,7 @@ class ChromaStore: """If inherited from BaseStore, or importing other modules from metagpt, a Python exception occurs, which is strange.""" + def __init__(self, name): client = chromadb.Client() collection = client.create_collection(name) @@ -22,7 +23,7 @@ def search(self, query, n_results=2, metadata_filter=None, document_filter=None) query_texts=[query], n_results=n_results, where=metadata_filter, # optional filter - where_document=document_filter # optional filter + where_document=document_filter, # optional filter ) return results diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py index e4b9473c7..c59056312 100644 --- a/metagpt/document_store/document.py +++ b/metagpt/document_store/document.py @@ -24,20 +24,20 @@ def validate_cols(content_col: str, df: pd.DataFrame): def read_data(data_path: Path): suffix = data_path.suffix - if '.xlsx' == suffix: + if ".xlsx" == suffix: data = pd.read_excel(data_path) - elif '.csv' == suffix: + elif ".csv" == suffix: data = pd.read_csv(data_path) - elif '.json' == suffix: + elif ".json" == suffix: data = pd.read_json(data_path) - elif suffix in ('.docx', '.doc'): - data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() - elif '.txt' == suffix: + elif suffix in (".docx", ".doc"): + data = UnstructuredWordDocumentLoader(str(data_path), mode="elements").load() + elif ".txt" == suffix: data = TextLoader(str(data_path)).load() - text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) + text_splitter = CharacterTextSplitter(separator="\n", chunk_size=256, chunk_overlap=0) texts = text_splitter.split_documents(data) data = texts - elif '.pdf' == suffix: + elif ".pdf" == suffix: data = UnstructuredPDFLoader(str(data_path), mode="elements").load() else: raise NotImplementedError @@ -45,8 +45,7 @@ def read_data(data_path: Path): class Document: - - def __init__(self, data_path, content_col='content', meta_col='metadata'): + def __init__(self, data_path, content_col="content", meta_col="metadata"): self.data = read_data(data_path) if isinstance(self.data, pd.DataFrame): validate_cols(content_col, self.data) @@ -79,4 +78,3 @@ def get_docs_and_metadatas(self) -> (list, list): return self._get_docs_and_metadatas_by_langchain() else: raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index dd450010d..8ff904cdd 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -20,7 +20,7 @@ class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'): + def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output"): self.meta_col = meta_col self.content_col = content_col super().__init__(raw_data, cache_dir) @@ -50,7 +50,7 @@ def persist(self): pickle.dump(store, f) store.index = index - def search(self, query, expand_cols=False, sep='\n', *args, k=5, **kwargs): + def search(self, query, expand_cols=False, sep="\n", *args, k=5, **kwargs): rsp = self.store.similarity_search(query, k=k, **kwargs) logger.debug(rsp) if expand_cols: @@ -78,8 +78,8 @@ def delete(self, *args, **kwargs): raise NotImplementedError -if __name__ == '__main__': - faiss_store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json') - logger.info(faiss_store.search('Oily Skin Facial Cleanser')) - faiss_store.add([f'Oily Skin Facial Cleanser-{i}' for i in range(3)]) - logger.info(faiss_store.search('Oily Skin Facial Cleanser')) +if __name__ == "__main__": + faiss_store = FaissStore(DATA_PATH / "qcs/qcs_4w.json") + logger.info(faiss_store.search("Oily Skin Facial Cleanser")) + faiss_store.add([f"Oily Skin Facial Cleanser-{i}" for i in range(3)]) + logger.info(faiss_store.search("Oily Skin Facial Cleanser")) diff --git a/metagpt/document_store/milvus_store.py b/metagpt/document_store/milvus_store.py index 77a8ec141..fcfc59d79 100644 --- a/metagpt/document_store/milvus_store.py +++ b/metagpt/document_store/milvus_store.py @@ -12,12 +12,7 @@ from metagpt.document_store.base_store import BaseStore -type_mapping = { - int: DataType.INT64, - str: DataType.VARCHAR, - float: DataType.DOUBLE, - np.ndarray: DataType.FLOAT_VECTOR -} +type_mapping = {int: DataType.INT64, str: DataType.VARCHAR, float: DataType.DOUBLE, np.ndarray: DataType.FLOAT_VECTOR} def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: str = ""): @@ -52,17 +47,11 @@ def __init__(self, connection): self.collection = None def _create_collection(self, name, schema): - collection = Collection( - name=name, - schema=schema, - using='default', - shards_num=2, - consistency_level="Strong" - ) + collection = Collection(name=name, schema=schema, using="default", shards_num=2, consistency_level="Strong") return collection def create_collection(self, name, columns): - schema = columns_to_milvus_schema(columns, 'idx') + schema = columns_to_milvus_schema(columns, "idx") self.collection = self._create_collection(name, schema) return self.collection @@ -72,7 +61,7 @@ def drop(self, name): def load_collection(self): self.collection.load() - def build_index(self, field='emb'): + def build_index(self, field="emb"): self.collection.create_index(field, {"index_type": "FLAT", "metric_type": "L2", "params": {}}) def search(self, query: list[list[float]], *args, **kwargs): @@ -85,11 +74,11 @@ def search(self, query: list[list[float]], *args, **kwargs): search_params = {"metric_type": "L2", "params": {"nprobe": 10}} results = self.collection.search( data=query, - anns_field=kwargs.get('field', 'emb'), + anns_field=kwargs.get("field", "emb"), param=search_params, limit=10, expr=None, - consistency_level="Strong" + consistency_level="Strong", ) # FIXME: results contain id, but to get the actual value from the id, we still need to call the query interface return results diff --git a/metagpt/document_store/qdrant_store.py b/metagpt/document_store/qdrant_store.py index 98b82cf87..4e9637aa7 100644 --- a/metagpt/document_store/qdrant_store.py +++ b/metagpt/document_store/qdrant_store.py @@ -10,13 +10,14 @@ @dataclass class QdrantConnection: """ - Args: - url: qdrant url - host: qdrant host - port: qdrant port - memory: qdrant service use memory mode - api_key: qdrant cloud api_key - """ + Args: + url: qdrant url + host: qdrant host + port: qdrant port + memory: qdrant service use memory mode + api_key: qdrant cloud api_key + """ + url: str = None host: str = None port: int = None @@ -31,9 +32,7 @@ def __init__(self, connect: QdrantConnection): elif connect.url: self.client = QdrantClient(url=connect.url, api_key=connect.api_key) elif connect.host and connect.port: - self.client = QdrantClient( - host=connect.host, port=connect.port, api_key=connect.api_key - ) + self.client = QdrantClient(host=connect.host, port=connect.port, api_key=connect.api_key) else: raise Exception("please check QdrantConnection.") @@ -58,15 +57,11 @@ def create_collection( try: self.client.get_collection(collection_name) if force_recreate: - res = self.client.recreate_collection( - collection_name, vectors_config=vectors_config, **kwargs - ) + res = self.client.recreate_collection(collection_name, vectors_config=vectors_config, **kwargs) return res return True except: # noqa: E722 - return self.client.recreate_collection( - collection_name, vectors_config=vectors_config, **kwargs - ) + return self.client.recreate_collection(collection_name, vectors_config=vectors_config, **kwargs) def has_collection(self, collection_name: str): try: diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..2e2aa152a 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -17,34 +17,34 @@ class Environment(BaseModel): """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 - Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles + """ roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) - history: str = Field(default='') + history: str = Field(default="") class Config: arbitrary_types_allowed = True def add_role(self, role: Role): """增加一个在当前环境的角色 - Add a role in the current environment + Add a role in the current environment """ role.set_env(self) self.roles[role.profile] = role def add_roles(self, roles: Iterable[Role]): """增加一批在当前环境的角色 - Add a batch of characters in the current environment + Add a batch of characters in the current environment """ for role in roles: self.add_role(role) def publish_message(self, message: Message): """向当前环境发布信息 - Post information to the current environment + Post information to the current environment """ # self.message_queue.put(message) self.memory.add(message) @@ -68,12 +68,12 @@ async def run(self, k=1): def get_roles(self) -> dict[str, Role]: """获得环境内的所有角色 - Process all Role runs at once + Process all Role runs at once """ return self.roles def get_role(self, name: str) -> Role: """获得环境内的指定角色 - get all the environment roles + get all the environment roles """ return self.roles.get(name, None) diff --git a/metagpt/inspect_module.py b/metagpt/inspect_module.py index a89ac1c5e..48ceffc57 100644 --- a/metagpt/inspect_module.py +++ b/metagpt/inspect_module.py @@ -12,17 +12,17 @@ def print_classes_and_functions(module): - """FIXME: NOT WORK.. """ + """FIXME: NOT WORK..""" for name, obj in inspect.getmembers(module): if inspect.isclass(obj): - print(f'Class: {name}') + print(f"Class: {name}") elif inspect.isfunction(obj): - print(f'Function: {name}') + print(f"Function: {name}") else: print(name) print(dir(module)) -if __name__ == '__main__': - print_classes_and_functions(metagpt) \ No newline at end of file +if __name__ == "__main__": + print_classes_and_functions(metagpt) diff --git a/metagpt/llm.py b/metagpt/llm.py index e6f815950..410f3dcb5 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -12,8 +12,9 @@ DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() + async def ai_func(prompt): """使用LLM进行QA - QA with LLMs - """ + QA with LLMs + """ return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/logs.py b/metagpt/logs.py index b2052e9b8..55d85312f 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -12,13 +12,15 @@ from metagpt.const import PROJECT_ROOT + def define_log_level(print_level="INFO", logfile_level="DEBUG"): """调整日志级别到level之上 - Adjust the log level to above level + Adjust the log level to above level """ _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) + _logger.add(PROJECT_ROOT / "logs/log.txt", level=logfile_level) return _logger + logger = define_log_level() diff --git a/metagpt/management/skill_manager.py b/metagpt/management/skill_manager.py index f967a0a94..b3181b64e 100644 --- a/metagpt/management/skill_manager.py +++ b/metagpt/management/skill_manager.py @@ -19,8 +19,8 @@ class SkillManager: def __init__(self): self._llm = LLM() - self._store = ChromaStore('skill_manager') - self._skills: dict[str: Skill] = {} + self._store = ChromaStore("skill_manager") + self._skills: dict[str:Skill] = {} def add_skill(self, skill: Skill): """ @@ -54,7 +54,7 @@ def retrieve_skill(self, desc: str, n_results: int = 2) -> list[Skill]: :param desc: Skill description :return: Multiple skills """ - return self._store.search(desc, n_results=n_results)['ids'][0] + return self._store.search(desc, n_results=n_results)["ids"][0] def retrieve_skill_scored(self, desc: str, n_results: int = 2) -> dict: """ @@ -75,6 +75,6 @@ def generate_skill_desc(self, skill: Skill) -> str: logger.info(text) -if __name__ == '__main__': +if __name__ == "__main__": manager = SkillManager() manager.generate_skill_desc(Action()) diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..d0b6b101c 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -18,7 +18,7 @@ def __init__(self, llm: LLM = LLM()): "Product Manager": "Architect", "Architect": "Engineer", "Engineer": "QA Engineer", - "QA Engineer": "Product Manager" + "QA Engineer": "Product Manager", } self.prompt_template = """ Given the following message: @@ -51,7 +51,7 @@ async def handle(self, message: Message, environment): # chosen_role_name = self.llm.ask(self.prompt_template.format(context)) # FIXME: 现在通过简单的字典决定流向，但之后还是应该有思考过程 - #The direction of flow is now determined by a simple dictionary, but there should still be a thought process afterwards + # The direction of flow is now determined by a simple dictionary, but there should still be a thought process afterwards next_role_profile = self.role_directions[message.role] # logger.debug(f"{next_role_profile}") for _, role in roles.items(): diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index f8abea5f3..e0b8e68c1 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -68,4 +68,3 @@ def delete(self, message: Message): def clear(self): super(LongTermMemory, self).clear() self.memory_storage.clean() - \ No newline at end of file diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index c818fa707..282f5fe33 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -85,4 +85,3 @@ def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: continue rsp += self.index[action] return rsp - \ No newline at end of file diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index 302d96aa7..a213f6d7a 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -2,16 +2,16 @@ # -*- coding: utf-8 -*- # @Desc : the implement of memory storage -from typing import List from pathlib import Path +from typing import List from langchain.vectorstores.faiss import FAISS from metagpt.const import DATA_PATH, MEM_TTL +from metagpt.document_store.faiss_store import FaissStore from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.serialize import serialize_message, deserialize_message -from metagpt.document_store.faiss_store import FaissStore +from metagpt.utils.serialize import deserialize_message, serialize_message class MemoryStorage(FaissStore): @@ -34,7 +34,7 @@ def is_initialized(self) -> bool: def recover_memory(self, role_id: str) -> List[Message]: self.role_id = role_id - self.role_mem_path = Path(DATA_PATH / f'role_mem/{self.role_id}/') + self.role_mem_path = Path(DATA_PATH / f"role_mem/{self.role_id}/") self.role_mem_path.mkdir(parents=True, exist_ok=True) self.store = self._load() @@ -51,18 +51,18 @@ def recover_memory(self, role_id: str) -> List[Message]: def _get_index_and_store_fname(self): if not self.role_mem_path: - logger.error(f'You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory') + logger.error(f"You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory") return None, None - index_fpath = Path(self.role_mem_path / f'{self.role_id}.index') - storage_fpath = Path(self.role_mem_path / f'{self.role_id}.pkl') + index_fpath = Path(self.role_mem_path / f"{self.role_id}.index") + storage_fpath = Path(self.role_mem_path / f"{self.role_id}.pkl") return index_fpath, storage_fpath def persist(self): super(MemoryStorage, self).persist() - logger.debug(f'Agent {self.role_id} persist memory into local') + logger.debug(f"Agent {self.role_id} persist memory into local") def add(self, message: Message) -> bool: - """ add message into memory storage""" + """add message into memory storage""" docs = [message.content] metadatas = [{"message_ser": serialize_message(message)}] if not self.store: @@ -79,10 +79,7 @@ def search_dissimilar(self, message: Message, k=4) -> List[Message]: if not self.store: return [] - resp = self.store.similarity_search_with_score( - query=message.content, - k=k - ) + resp = self.store.similarity_search_with_score(query=message.content, k=k) # filter the result which score is smaller than the threshold filtered_resp = [] for item, score in resp: @@ -104,4 +101,3 @@ def clean(self): self.store = None self._initialized = False - \ No newline at end of file diff --git a/metagpt/prompts/invoice_ocr.py b/metagpt/prompts/invoice_ocr.py index 52f628a5b..aa79651be 100644 --- a/metagpt/prompts/invoice_ocr.py +++ b/metagpt/prompts/invoice_ocr.py @@ -10,7 +10,9 @@ COMMON_PROMPT = "Now I will provide you with the OCR text recognition results for the invoice." -EXTRACT_OCR_MAIN_INFO_PROMPT = COMMON_PROMPT + """ +EXTRACT_OCR_MAIN_INFO_PROMPT = ( + COMMON_PROMPT + + """ Please extract the payee, city, total cost, and invoicing date of the invoice. The OCR data of the invoice are as follows: @@ -22,8 +24,11 @@ 2. The returned JSON dictionary must be returned in {language} 3. Mandatory requirement to output in JSON format: {{"收款人":"x","城市":"x","总费用/元":"","开票日期":""}}. """ +) -REPLY_OCR_QUESTION_PROMPT = COMMON_PROMPT + """ +REPLY_OCR_QUESTION_PROMPT = ( + COMMON_PROMPT + + """ Please answer the question: {query} The OCR data of the invoice are as follows: @@ -34,6 +39,6 @@ 2. Enforce restrictions on not returning OCR data sent to you. 3. Return with markdown syntax layout. """ +) INVOICE_OCR_SUCCESS = "Successfully completed OCR text recognition invoice." - diff --git a/metagpt/prompts/sales.py b/metagpt/prompts/sales.py index a44aacafe..30ef1ae02 100644 --- a/metagpt/prompts/sales.py +++ b/metagpt/prompts/sales.py @@ -54,10 +54,12 @@ {salesperson_name}: """ -conversation_stages = {'1' : "Introduction: Start the conversation by introducing yourself and your company. Be polite and respectful while keeping the tone of the conversation professional. Your greeting should be welcoming. Always clarify in your greeting the reason why you are contacting the prospect.", -'2': "Qualification: Qualify the prospect by confirming if they are the right person to talk to regarding your product/service. Ensure that they have the authority to make purchasing decisions.", -'3': "Value proposition: Briefly explain how your product/service can benefit the prospect. Focus on the unique selling points and value proposition of your product/service that sets it apart from competitors.", -'4': "Needs analysis: Ask open-ended questions to uncover the prospect's needs and pain points. Listen carefully to their responses and take notes.", -'5': "Solution presentation: Based on the prospect's needs, present your product/service as the solution that can address their pain points.", -'6': "Objection handling: Address any objections that the prospect may have regarding your product/service. Be prepared to provide evidence or testimonials to support your claims.", -'7': "Close: Ask for the sale by proposing a next step. This could be a demo, a trial or a meeting with decision-makers. Ensure to summarize what has been discussed and reiterate the benefits."} +conversation_stages = { + "1": "Introduction: Start the conversation by introducing yourself and your company. Be polite and respectful while keeping the tone of the conversation professional. Your greeting should be welcoming. Always clarify in your greeting the reason why you are contacting the prospect.", + "2": "Qualification: Qualify the prospect by confirming if they are the right person to talk to regarding your product/service. Ensure that they have the authority to make purchasing decisions.", + "3": "Value proposition: Briefly explain how your product/service can benefit the prospect. Focus on the unique selling points and value proposition of your product/service that sets it apart from competitors.", + "4": "Needs analysis: Ask open-ended questions to uncover the prospect's needs and pain points. Listen carefully to their responses and take notes.", + "5": "Solution presentation: Based on the prospect's needs, present your product/service as the solution that can address their pain points.", + "6": "Objection handling: Address any objections that the prospect may have regarding your product/service. Be prepared to provide evidence or testimonials to support your claims.", + "7": "Close: Ask for the sale by proposing a next step. This could be a demo, a trial or a meeting with decision-makers. Ensure to summarize what has been discussed and reiterate the benefits.", +} diff --git a/metagpt/prompts/tutorial_assistant.py b/metagpt/prompts/tutorial_assistant.py index d690aad83..3d4b6fa24 100644 --- a/metagpt/prompts/tutorial_assistant.py +++ b/metagpt/prompts/tutorial_assistant.py @@ -12,7 +12,9 @@ We need you to write a technical tutorial with the topic "{topic}". """ -DIRECTORY_PROMPT = COMMON_PROMPT + """ +DIRECTORY_PROMPT = ( + COMMON_PROMPT + + """ Please provide the specific table of contents for this tutorial, strictly following the following requirements: 1. The output must be strictly in the specified language, {language}. 2. Answer strictly in the dictionary format like {{"title": "xxx", "directory": [{{"dir 1": ["sub dir 1", "sub dir 2"]}}, {{"dir 2": ["sub dir 3", "sub dir 4"]}}]}}. @@ -20,8 +22,11 @@ 4. Do not have extra spaces or line breaks. 5. Each directory title has practical significance. """ +) -CONTENT_PROMPT = COMMON_PROMPT + """ +CONTENT_PROMPT = ( + COMMON_PROMPT + + """ Now I will give you the module directory titles for the topic. Please output the detailed principle content of this title in detail. If there are code examples, please provide them according to standard code specifications. @@ -36,4 +41,5 @@ 3. The output must be strictly in the specified language, {language}. 4. Do not have redundant output, including concluding remarks. 5. Strict requirement not to output the topic "{topic}". -""" \ No newline at end of file +""" +) diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 7293e2cde..03802a716 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -32,4 +32,3 @@ async def aask(self, prompt): max_tokens_to_sample=1000, ) return res.completion - \ No newline at end of file diff --git a/metagpt/provider/base_chatbot.py b/metagpt/provider/base_chatbot.py index abdf423f4..2d4cfe2d9 100644 --- a/metagpt/provider/base_chatbot.py +++ b/metagpt/provider/base_chatbot.py @@ -12,6 +12,7 @@ @dataclass class BaseChatbot(ABC): """Abstract GPT class""" + mode: str = "API" @abstractmethod @@ -25,4 +26,3 @@ def ask_batch(self, msgs: list) -> str: @abstractmethod def ask_code(self, msgs: list) -> str: """Ask GPT multiple questions and get a piece of code""" - \ No newline at end of file diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index de61167b9..adc57c66b 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -14,7 +14,8 @@ class BaseGPTAPI(BaseChatbot): """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities""" - system_prompt = 'You are a helpful assistant.' + + system_prompt = "You are a helpful assistant." def _user_msg(self, msg: str) -> dict[str, str]: return {"role": "user", "content": msg} @@ -110,9 +111,8 @@ def get_choice_text(self, rsp: dict) -> str: def messages_to_prompt(self, messages: list[dict]): """[{"role": "user", "content": msg}] to user: etc.""" - return '\n'.join([f"{i['role']}: {i['content']}" for i in messages]) + return "\n".join([f"{i['role']}: {i['content']}" for i in messages]) def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" return [i.to_dict() for i in messages] - \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6ebed2c16..ac0edd44f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -110,7 +110,6 @@ def get_total_completion_tokens(self): """ return self.total_completion_tokens - def get_total_cost(self): """ Get the total cost of API calls. @@ -120,7 +119,6 @@ def get_total_cost(self): """ return self.total_cost - def get_costs(self) -> Costs: """Get all costs""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) diff --git a/metagpt/provider/spark_api.py b/metagpt/provider/spark_api.py index 55f7000ec..60c86f4dc 100644 --- a/metagpt/provider/spark_api.py +++ b/metagpt/provider/spark_api.py @@ -14,8 +14,7 @@ import ssl from time import mktime from typing import Optional -from urllib.parse import urlencode -from urllib.parse import urlparse +from urllib.parse import urlencode, urlparse from wsgiref.handlers import format_date_time import websocket # 使用websocket_client @@ -26,9 +25,8 @@ class SparkAPI(BaseGPTAPI): - def __init__(self): - logger.warning('当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。') + logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。") def ask(self, msg: str) -> str: message = [self._default_system_msg(), self._user_msg(msg)] @@ -49,7 +47,7 @@ def get_choice_text(self, rsp: dict) -> str: async def acompletion_text(self, messages: list[dict], stream=False) -> str: # 不支持 - logger.error('该功能禁用。') + logger.error("该功能禁用。") w = GetMessageFromWeb(messages) return w.run() @@ -93,29 +91,26 @@ def create_url(self): signature_origin += "GET " + self.path + " HTTP/1.1" # 进行hmac-sha256进行加密 - signature_sha = hmac.new(self.api_secret.encode('utf-8'), signature_origin.encode('utf-8'), - digestmod=hashlib.sha256).digest() + signature_sha = hmac.new( + self.api_secret.encode("utf-8"), signature_origin.encode("utf-8"), digestmod=hashlib.sha256 + ).digest() - signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding='utf-8') + signature_sha_base64 = base64.b64encode(signature_sha).decode(encoding="utf-8") authorization_origin = f'api_key="{self.api_key}", algorithm="hmac-sha256", headers="host date request-line", signature="{signature_sha_base64}"' - authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8') + authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(encoding="utf-8") # 将请求的鉴权参数组合为字典 - v = { - "authorization": authorization, - "date": date, - "host": self.host - } + v = {"authorization": authorization, "date": date, "host": self.host} # 拼接鉴权参数，生成url - url = self.spark_url + '?' + urlencode(v) + url = self.spark_url + "?" + urlencode(v) # 此处打印出建立连接时候的url,参考本demo的时候可取消上方打印的注释，比对相同参数时生成的url与自己代码生成的url是否一致 return url def __init__(self, text): self.text = text - self.ret = '' + self.ret = "" self.spark_appid = CONFIG.spark_appid self.spark_api_secret = CONFIG.spark_api_secret self.spark_api_key = CONFIG.spark_api_key @@ -124,15 +119,15 @@ def __init__(self, text): def on_message(self, ws, message): data = json.loads(message) - code = data['header']['code'] + code = data["header"]["code"] if code != 0: ws.close() # 请求错误，则关闭socket - logger.critical(f'回答获取失败，响应信息反序列化之后为： {data}') + logger.critical(f"回答获取失败，响应信息反序列化之后为： {data}") return else: choices = data["payload"]["choices"] - seq = choices["seq"] # 服务端是流式返回，seq为返回的数据序号 + # seq = choices["seq"] # 服务端是流式返回，seq为返回的数据序号 status = choices["status"] # 服务端是流式返回，status用于判断信息是否传送完毕 content = choices["text"][0]["content"] # 本次接收到的回答文本 self.ret += content @@ -142,7 +137,7 @@ def on_message(self, ws, message): # 收到websocket错误的处理 def on_error(self, ws, error): # on_message方法处理接收到的信息，出现任何错误，都会调用这个方法 - logger.critical(f'通讯连接出错，【错误提示: {error}】') + logger.critical(f"通讯连接出错，【错误提示: {error}】") # 收到websocket关闭的处理 def on_close(self, ws, one, two): @@ -150,17 +145,12 @@ def on_close(self, ws, one, two): # 处理请求数据 def gen_params(self): - data = { - "header": { - "app_id": self.spark_appid, - "uid": "1234" - }, + "header": {"app_id": self.spark_appid, "uid": "1234"}, "parameter": { "chat": { # domain为必传参数 "domain": self.domain, - # 以下为可微调，非必传参数 # 注意：官方建议，temperature和top_k修改一个即可 "max_tokens": 2048, # 默认2048，模型回答的tokens的最大长度，即允许它输出文本的最长字数 @@ -168,11 +158,7 @@ def gen_params(self): "top_k": 4, # 取值为[1，6],默认为4。从k个候选中随机选择一个（非等概率） } }, - "payload": { - "message": { - "text": self.text - } - } + "payload": {"message": {"text": self.text}}, } return data @@ -189,17 +175,12 @@ def run(self): return self._run(self.text) def _run(self, text_list): - - ws_param = self.WsParam( - self.spark_appid, - self.spark_api_key, - self.spark_api_secret, - self.spark_url, - text_list) + ws_param = self.WsParam(self.spark_appid, self.spark_api_key, self.spark_api_secret, self.spark_url, text_list) ws_url = ws_param.create_url() websocket.enableTrace(False) # 默认禁用 WebSocket 的跟踪功能 - ws = websocket.WebSocketApp(ws_url, on_message=self.on_message, on_error=self.on_error, on_close=self.on_close, - on_open=self.on_open) + ws = websocket.WebSocketApp( + ws_url, on_message=self.on_message, on_error=self.on_error, on_close=self.on_close, on_open=self.on_open + ) ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE}) return self.ret diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index 4547f8190..188182d47 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -24,12 +24,5 @@ class CustomerService(Sales): - def __init__( - self, - name="Xiaomei", - profile="Human customer service", - desc=DESC, - store=None - ): + def __init__(self, name="Xiaomei", profile="Human customer service", desc=DESC, store=None): super().__init__(name, profile, desc=desc, store=store) - \ No newline at end of file diff --git a/metagpt/roles/invoice_ocr_assistant.py b/metagpt/roles/invoice_ocr_assistant.py index c307b20c0..3087a4da7 100644 --- a/metagpt/roles/invoice_ocr_assistant.py +++ b/metagpt/roles/invoice_ocr_assistant.py @@ -9,7 +9,7 @@ import pandas as pd -from metagpt.actions.invoice_ocr import InvoiceOCR, GenerateTable, ReplyQuestion +from metagpt.actions.invoice_ocr import GenerateTable, InvoiceOCR, ReplyQuestion from metagpt.prompts.invoice_ocr import INVOICE_OCR_SUCCESS from metagpt.roles import Role from metagpt.schema import Message @@ -107,4 +107,3 @@ async def _react(self) -> Message: break msg = await self._act() return msg - diff --git a/metagpt/roles/prompt.py b/metagpt/roles/prompt.py index c22e0226b..457ccb6c6 100644 --- a/metagpt/roles/prompt.py +++ b/metagpt/roles/prompt.py @@ -23,6 +23,7 @@ Question: {input} Thoughts: {agent_scratchpad}""" + class PromptString(Enum): REFLECTION_QUESTIONS = "Here are some statements:\n{memory_descriptions}\n\nBased solely on the information above, what are the 3 most prominent high-level questions we can answer about the topic in the statements?\n\n{format_instructions}" @@ -32,7 +33,7 @@ class PromptString(Enum): RECENT_ACTIVITY = "Based on the following memory, produce a brief summary of what {full_name} has been up to recently. Do not invent details not explicitly stated in the memory. For any conversation, be sure to mention whether the conversation has concluded or is still ongoing.\n\nMemory: {memory_descriptions}" - MAKE_PLANS = "You are a plan-generating AI. Your job is to assist the character in formulating new plans based on new information. Given the character's information (profile, objectives, recent activities, current plans, and location context) and their current thought process, produce a new set of plans for them. The final plan should comprise at least {time_window} of activities and no more than 5 individual plans. List the plans in the order they should be executed, with each plan detailing its description, location, start time, stop criteria, and maximum duration.\n\nSample plan: {{\"index\": 1, \"description\": \"Cook dinner\", \"location_id\": \"0a3bc22b-36aa-48ab-adb0-18616004caed\",\"start_time\": \"2022-12-12T20:00:00+00:00\",\"max_duration_hrs\": 1.5, \"stop_condition\": \"Dinner is fully prepared\"}}\'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nIt's essential to encourage the character to collaborate with other characters in their plans.\n\n" + MAKE_PLANS = 'You are a plan-generating AI. Your job is to assist the character in formulating new plans based on new information. Given the character\'s information (profile, objectives, recent activities, current plans, and location context) and their current thought process, produce a new set of plans for them. The final plan should comprise at least {time_window} of activities and no more than 5 individual plans. List the plans in the order they should be executed, with each plan detailing its description, location, start time, stop criteria, and maximum duration.\n\nSample plan: {{"index": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}\'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet\'s begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nIt\'s essential to encourage the character to collaborate with other characters in their plans.\n\n' EXECUTE_PLAN = "You are a role-playing AI, playing the role of {your_name}, in front of a live audience. Every word you say can be observed by the audience, so make sure you speak often and make it interesting. You cannot interact directly with the audience.\n\nBased on the following context and tools, proceed as if you are {your_name}. Your primary task is to complete the task given below, however, if you are currently in a conversation with another character, you should always complete the conversation first and then proceed with the task. Do not start working while you are engaged in an unfinished conversation. Use your best judgment to determine whether a conversation involves you and whether it is unfinished. You don't need to respond to every message you receive.\n\nThis is a personal profile of your character:\n{your_private_bio}\n\nThis is a description of your location and other nearby characters you can converse with:\n\n{location_context}\n\nThese are some relevant memories:\n```\n{relevant_memories}\n```\n\nThis is some relevant conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour response should follow the format below:\n\nTask: The task you must complete\nThoughts: What you should think about doing\nAction: The action to take, must be one of these words: [{tool_names}]\nAction Input: Input for the action\nObservation: The result of the action\n... (This Thoughts/Action/Action Input/Observation can be repeated N times)\nThoughts: 'I have completed the task'\nFinal Response: The final response to the task\n\nIf you are not ready with a final response, then you must take an action.\n\nIf you are certain that you cannot complete the task with the tools provided, return 'Final Response: Need Help', however, if you are in conversation with another character, responses like 'I don't know' are valid. During conversations, you should never break character or admit you are an AI.\nIf the task is completed and no specific response is required, return 'Final Response: Done'\nLet's begin!\n\nTask: {input}\n\n{agent_scratchpad}" @@ -40,6 +41,6 @@ class PromptString(Enum): GOSSIP = "You are {full_name}. \n{memory_descriptions}\n\nBased on the statements above, say a thing or two of interest to others at your location: {other_agent_names}.\nAlways specify their names when referring to others." - HAS_HAPPENED = "Given the descriptions of the observations of the following characters and the events they are awaiting, indicate whether the character has witnessed the event.\n{format_instructions}\n\nExample:\n\nObservations:\nJoe entered the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe made some breakfast at 2023-05-04 08:15:00+00:00\n\nAwaiting: Sally responded to Joe\n\nYour response: '{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\nLet's begin!\n\nObservations:\n{memory_descriptions}\n\nAwaiting: {event_description}\n" + HAS_HAPPENED = 'Given the descriptions of the observations of the following characters and the events they are awaiting, indicate whether the character has witnessed the event.\n{format_instructions}\n\nExample:\n\nObservations:\nJoe entered the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe made some breakfast at 2023-05-04 08:15:00+00:00\n\nAwaiting: Sally responded to Joe\n\nYour response: \'{{"has_happened": true, "date_occured": 2023-05-04 08:05:30+00:00}}\'\n\nLet\'s begin!\n\nObservations:\n{memory_descriptions}\n\nAwaiting: {event_description}\n' OUTPUT_FORMAT = "\n\n(Remember! Make sure your output always adheres to one of the following two formats:\n\nA. If you have completed the task:\nThoughts: 'I have completed the task'\nFinal Response: \n\nB. If you haven't completed the task:\nThoughts: \nAction: \nAction Input: \nObservation: )\n" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 44bb3e976..282431bf7 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -11,12 +11,13 @@ from pydantic import BaseModel, Field +from metagpt.actions import Action, ActionOutput + # from metagpt.environment import Environment from metagpt.config import CONFIG -from metagpt.actions import Action, ActionOutput from metagpt.llm import LLM from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -49,6 +50,7 @@ class RoleSetting(BaseModel): """Role Settings""" + name: str profile: str goal: str @@ -64,7 +66,8 @@ def __repr__(self): class RoleContext(BaseModel): """Role Runtime Context""" - env: 'Environment' = Field(default=None) + + env: "Environment" = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=0) @@ -128,7 +131,7 @@ def _set_state(self, state): logger.debug(self._actions) self._rc.todo = self._actions[self._rc.state] - def set_env(self, env: 'Environment'): + def set_env(self, env: "Environment"): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env @@ -150,12 +153,13 @@ async def _think(self) -> None: self._set_state(0) return prompt = self._get_prefix() - prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + prompt += STATE_TEMPLATE.format( + history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1 + ) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') + logger.warning(f"Invalid answer of state, {next_state=}") next_state = "0" self._set_state(int(next_state)) @@ -168,8 +172,12 @@ async def _act(self) -> Message: response = await self._rc.todo.run(self._rc.important_memory) # logger.info(response) if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=type(self._rc.todo), + ) else: msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) @@ -184,15 +192,17 @@ async def _observe(self) -> int: env_msgs = self._rc.env.memory.get() observed = self._rc.env.memory.get_by_actions(self._rc.watch) - - self._rc.news = self._rc.memory.find_news(observed) # find news (previously unseen messages) from observed messages + + self._rc.news = self._rc.memory.find_news( + observed + ) # find news (previously unseen messages) from observed messages for i in env_msgs: self.recv(i) news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: - logger.debug(f'{self._setting} observed: {news_text}') + logger.debug(f"{self._setting} observed: {news_text}") return len(self._rc.news) def _publish_message(self, msg): diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index a45ad6f1b..18282a494 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -12,16 +12,16 @@ class Sales(Role): def __init__( - self, - name="Xiaomei", - profile="Retail sales guide", - desc="I am a sales guide in retail. My name is Xiaomei. I will answer some customer questions next, and I " - "will answer questions only based on the information in the knowledge base." - "If I feel that you can't get the answer from the reference material, then I will directly reply that" - " I don't know, and I won't tell you that this is from the knowledge base," - "but pretend to be what I know. Note that each of my replies will be replied in the tone of a " - "professional guide", - store=None + self, + name="Xiaomei", + profile="Retail sales guide", + desc="I am a sales guide in retail. My name is Xiaomei. I will answer some customer questions next, and I " + "will answer questions only based on the information in the knowledge base." + "If I feel that you can't get the answer from the reference material, then I will directly reply that" + " I don't know, and I won't tell you that this is from the knowledge base," + "but pretend to be what I know. Note that each of my replies will be replied in the tone of a " + "professional guide", + store=None, ): super().__init__(name, profile, desc=desc) self._set_store(store) @@ -32,4 +32,3 @@ def _set_store(self, store): else: action = SearchAndSummarize() self._init_actions([action]) - \ No newline at end of file diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index 0b6e089da..a2c4896e2 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -15,7 +15,7 @@ class Searcher(Role): """ Represents a Searcher role responsible for providing search services to users. - + Attributes: name (str): Name of the searcher. profile (str): Role profile. @@ -23,17 +23,19 @@ class Searcher(Role): constraints (str): Constraints or limitations for the searcher. engine (SearchEngineType): The type of search engine to use. """ - - def __init__(self, - name: str = 'Alice', - profile: str = 'Smart Assistant', - goal: str = 'Provide search services for users', - constraints: str = 'Answer is rich and complete', - engine=SearchEngineType.SERPAPI_GOOGLE, - **kwargs) -> None: + + def __init__( + self, + name: str = "Alice", + profile: str = "Smart Assistant", + goal: str = "Provide search services for users", + constraints: str = "Answer is rich and complete", + engine=SearchEngineType.SERPAPI_GOOGLE, + **kwargs, + ) -> None: """ Initializes the Searcher role with given attributes. - + Args: name (str): Name of the searcher. profile (str): Role profile. @@ -53,10 +55,14 @@ async def _act_sp(self) -> Message: """Performs the search action in a single process.""" logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.memory.get(k=0)) - + if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=type(self._rc.todo), + ) else: msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) diff --git a/metagpt/roles/tutorial_assistant.py b/metagpt/roles/tutorial_assistant.py index 9a7df4f4d..2a514f433 100644 --- a/metagpt/roles/tutorial_assistant.py +++ b/metagpt/roles/tutorial_assistant.py @@ -9,7 +9,7 @@ from datetime import datetime from typing import Dict -from metagpt.actions.write_tutorial import WriteDirectory, WriteContent +from metagpt.actions.write_tutorial import WriteContent, WriteDirectory from metagpt.const import TUTORIAL_PATH from metagpt.logs import logger from metagpt.roles import Role @@ -110,5 +110,5 @@ async def _react(self) -> Message: break msg = await self._act() root_path = TUTORIAL_PATH / datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - await File.write(root_path, f"{self.main_title}.md", self.total_content.encode('utf-8')) + await File.write(root_path, f"{self.main_title}.md", self.total_content.encode("utf-8")) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index bdca093c2..19c7a6654 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -23,9 +23,10 @@ class RawMessage(TypedDict): @dataclass class Message: """list[: ]""" + content: str instruct_content: BaseModel = field(default=None) - role: str = field(default='user') # system / user / assistant + role: str = field(default="user") # system / user / assistant cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") @@ -39,45 +40,45 @@ def __repr__(self): return self.__str__() def to_dict(self) -> dict: - return { - "role": self.role, - "content": self.content - } + return {"role": self.role, "content": self.content} @dataclass class UserMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'user') + super().__init__(content, "user") @dataclass class SystemMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'system') + super().__init__(content, "system") @dataclass class AIMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ + def __init__(self, content: str): - super().__init__(content, 'assistant') + super().__init__(content, "assistant") -if __name__ == '__main__': - test_content = 'test_message' +if __name__ == "__main__": + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] logger.info(msgs) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index b2bd18c58..d3c2c463b 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -21,6 +21,7 @@ class SoftwareCompany(BaseModel): Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to writing executable code. """ + environment: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -36,11 +37,11 @@ def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment CONFIG.max_budget = investment - logger.info(f'Investment: ${investment}.') + logger.info(f"Investment: ${investment}.") def _check_balance(self): if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') + raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}") def start_project(self, idea): """Start a project from publishing boss requirement.""" @@ -59,4 +60,3 @@ async def run(self, n_round=3): self._check_balance() await self.environment.run() return self.environment.history - \ No newline at end of file diff --git a/metagpt/tools/code_interpreter.py b/metagpt/tools/code_interpreter.py index e41eaab72..1cba005fa 100644 --- a/metagpt/tools/code_interpreter.py +++ b/metagpt/tools/code_interpreter.py @@ -1,22 +1,26 @@ +import inspect import re -from typing import List, Callable, Dict +import textwrap from pathlib import Path +from typing import Callable, Dict, List import wrapt -import textwrap -import inspect from interpreter.core.core import Interpreter -from metagpt.logs import logger +from metagpt.actions.clone_function import ( + CloneFunction, + run_function_code, + run_function_script, +) from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.utils.highlight import highlight -from metagpt.actions.clone_function import CloneFunction, run_function_code, run_function_script def extract_python_code(code: str): """Extract code blocks: If the code comments are the same, only the last code block is kept.""" # Use regular expressions to match comment blocks and related code. - pattern = r'(#\s[^\n]*)\n(.*?)(?=\n\s*#|$)' + pattern = r"(#\s[^\n]*)\n(.*?)(?=\n\s*#|$)" matches = re.findall(pattern, code, re.DOTALL) # Extract the last code block when encountering the same comment. @@ -25,8 +29,8 @@ def extract_python_code(code: str): unique_comments[comment] = code_block # concatenate into functional form - result_code = '\n'.join([f"{comment}\n{code_block}" for comment, code_block in unique_comments.items()]) - header_code = code[:code.find("#")] + result_code = "\n".join([f"{comment}\n{code_block}" for comment, code_block in unique_comments.items()]) + header_code = code[: code.find("#")] code = header_code + result_code logger.info(f"Extract python code: \n {highlight(code)}") @@ -36,6 +40,7 @@ def extract_python_code(code: str): class OpenCodeInterpreter(object): """https://github.com/KillianLucas/open-interpreter""" + def __init__(self, auto_run: bool = True) -> None: interpreter = Interpreter() interpreter.auto_run = auto_run @@ -50,15 +55,16 @@ def chat(self, query: str, reset: bool = True): return self.interpreter.chat(query) @staticmethod - def extract_function(query_respond: List, function_name: str, *, language: str = 'python', - function_format: str = None) -> str: + def extract_function( + query_respond: List, function_name: str, *, language: str = "python", function_format: str = None + ) -> str: """create a function from query_respond.""" - if language not in ('python'): + if language not in ("python"): raise NotImplementedError(f"Not support to parse language {language}!") # set function form if function_format is None: - assert language == 'python', f"Expect python language for default function_format, but got {language}." + assert language == "python", f"Expect python language for default function_format, but got {language}." function_format = """def {function_name}():\n{code}""" # Extract the code module in the open-interpreter respond message. # The query_respond of open-interpreter before v0.1.4 is: @@ -68,25 +74,29 @@ def extract_function(query_respond: List, function_name: str, *, language: str = # "parsed_arguments": {"language": "python", "code": code of first plan} # ...] if "function_call" in query_respond[1]: - code = [item['function_call']['parsed_arguments']['code'] for item in query_respond - if "function_call" in item - and "parsed_arguments" in item["function_call"] - and 'language' in item["function_call"]['parsed_arguments'] - and item["function_call"]['parsed_arguments']['language'] == language] + code = [ + item["function_call"]["parsed_arguments"]["code"] + for item in query_respond + if "function_call" in item + and "parsed_arguments" in item["function_call"] + and "language" in item["function_call"]["parsed_arguments"] + and item["function_call"]["parsed_arguments"]["language"] == language + ] # The query_respond of open-interpreter v0.1.7 is: # [{'role': 'user', 'message': your query string}, # {'role': 'assistant', 'message': plan from llm, 'language': 'python', # 'code': code of first plan, 'output': output of first plan code}, # ...] elif "code" in query_respond[1]: - code = [item['code'] for item in query_respond - if "code" in item - and 'language' in item - and item['language'] == language] + code = [ + item["code"] + for item in query_respond + if "code" in item and "language" in item and item["language"] == language + ] else: raise ValueError(f"Unexpect message format in query_respond: {query_respond[1].keys()}") # add indent. - indented_code_str = textwrap.indent("\n".join(code), ' ' * 4) + indented_code_str = textwrap.indent("\n".join(code), " " * 4) # Return the code after deduplication. if language == "python": return extract_python_code(function_format.format(function_name=function_name, code=indented_code_str)) @@ -115,13 +125,13 @@ def __init__(self, save_code: bool = False, code_file_path: str = None, clear_co def _have_code(self, rsp: List[Dict]): # Is there any code generated? - return 'code' in rsp[1] and rsp[1]['code'] not in ("", None) + return "code" in rsp[1] and rsp[1]["code"] not in ("", None) def _is_faild_plan(self, rsp: List[Dict]): # is faild plan? - func_code = OpenCodeInterpreter.extract_function(rsp, 'function') + func_code = OpenCodeInterpreter.extract_function(rsp, "function") # If there is no more than 1 '\n', the plan execution fails. - if isinstance(func_code, str) and func_code.count('\n') <= 1: + if isinstance(func_code, str) and func_code.count("\n") <= 1: return True return False @@ -184,4 +194,5 @@ async def wrapper(wrapped: Callable, instance, args, kwargs): logger.error(f"Could not evaluate Python code \n{logger_code}: \nError: {e}") raise Exception("Could not evaluate Python code", e) return res + return wrapper(wrapped) diff --git a/metagpt/tools/prompt_writer.py b/metagpt/tools/prompt_writer.py index d90599206..ffcff4d1f 100644 --- a/metagpt/tools/prompt_writer.py +++ b/metagpt/tools/prompt_writer.py @@ -10,8 +10,9 @@ class GPTPromptGenerator: """Using LLM, given an output, request LLM to provide input (supporting instruction, chatbot, and query styles)""" + def __init__(self): - self._generators = {i: getattr(self, f"gen_{i}_style") for i in ['instruction', 'chatbot', 'query']} + self._generators = {i: getattr(self, f"gen_{i}_style") for i in ["instruction", "chatbot", "query"]} def gen_instruction_style(self, example): """Instruction style: Given an output, request LLM to provide input""" @@ -35,7 +36,7 @@ def gen_query_style(self, example): Document: {example} What is the detailed query X? X:""" - def gen(self, example: str, style: str = 'all') -> Union[list[str], str]: + def gen(self, example: str, style: str = "all") -> Union[list[str], str]: """ Generate one or multiple outputs using the example, allowing LLM to reply with the corresponding input @@ -43,7 +44,7 @@ def gen(self, example: str, style: str = 'all') -> Union[list[str], str]: :param style: (all|instruction|chatbot|query) :return: Expected LLM input sample (one or multiple) """ - if style != 'all': + if style != "all": return self._generators[style](example) return [f(example) for f in self._generators.values()] diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 1d9cd0b2a..a63dbe5ac 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -120,11 +120,13 @@ def decode_base64_to_image(img, save_name): image.save(f"{save_name}.png", pnginfo=pnginfo) return pnginfo, image + def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): for idx, _img in enumerate(imgs): save_name = join(save_dir, save_name) decode_base64_to_image(_img, save_name=save_name) + if __name__ == "__main__": engine = SDEngine() prompt = "pixel style, game design, a game interface should be minimalistic and intuitive with the score and high score displayed at the top. The snake and its food should be easily distinguishable. The game should have a simple color scheme, with a contrasting color for the snake and its food. Complete interface boundary" diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 942ef7edd..64388a11f 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -6,7 +6,7 @@ @File : search_engine.py """ import importlib -from typing import Callable, Coroutine, Literal, overload, Optional, Union +from typing import Callable, Coroutine, Literal, Optional, Union, overload from semantic_kernel.skill_definition import sk_function @@ -43,8 +43,8 @@ class SearchEngine: def __init__( self, - engine: Optional[SearchEngineType] = None, - run_func: Callable[[str, int, bool], Coroutine[None, None, Union[str, list[str]]]] = None, + engine: Optional[SearchEngineType] = None, + run_func: Callable[[str, int, bool], Coroutine[None, None, Union[str, list[str]]]] = None, ): engine = engine or CONFIG.search_engine if engine == SearchEngineType.SERPAPI_GOOGLE: diff --git a/metagpt/tools/search_engine_meilisearch.py b/metagpt/tools/search_engine_meilisearch.py index da4269384..f7c1c685a 100644 --- a/metagpt/tools/search_engine_meilisearch.py +++ b/metagpt/tools/search_engine_meilisearch.py @@ -29,7 +29,7 @@ def set_index(self, index): def add_documents(self, data_source: DataSource, documents: List[dict]): index_name = f"{data_source.name}_index" if index_name not in self.client.get_indexes(): - self.client.create_index(uid=index_name, options={'primaryKey': 'id'}) + self.client.create_index(uid=index_name, options={"primaryKey": "id"}) index = self.client.get_index(index_name) index.add_documents(documents) self.set_index(index) @@ -37,7 +37,7 @@ def add_documents(self, data_source: DataSource, documents: List[dict]): def search(self, query): try: search_results = self._index.search(query) - return search_results['hits'] + return search_results["hits"] except Exception as e: # Handle MeiliSearch API errors print(f"MeiliSearch API error: {e}") diff --git a/metagpt/tools/translator.py b/metagpt/tools/translator.py index 910638469..63e38d5a5 100644 --- a/metagpt/tools/translator.py +++ b/metagpt/tools/translator.py @@ -6,7 +6,7 @@ @File : translator.py """ -prompt = ''' +prompt = """ # 指令 接下来，作为一位拥有20年翻译经验的翻译专家，当我给出英文句子或段落时，你将提供通顺且具有可读性的{LANG}翻译。注意以下要求： 1. 确保翻译结果流畅且易于理解 @@ -17,11 +17,10 @@ {ORIGINAL} # 译文 -''' +""" class Translator: - @classmethod - def translate_prompt(cls, original, lang='中文'): - return prompt.format(LANG=lang, ORIGINAL=original) \ No newline at end of file + def translate_prompt(cls, original, lang="中文"): + return prompt.format(LANG=lang, ORIGINAL=original) diff --git a/metagpt/tools/ut_writer.py b/metagpt/tools/ut_writer.py index 43ca72150..64423dfb1 100644 --- a/metagpt/tools/ut_writer.py +++ b/metagpt/tools/ut_writer.py @@ -6,7 +6,7 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI -ICL_SAMPLE = '''Interface definition: +ICL_SAMPLE = """Interface definition: ```text Interface Name: Element Tagging Interface Path: /projects/{project_key}/node-tags @@ -60,20 +60,20 @@ def test_node_tags(project_key, nodes, operations, expected_msg): # 3. If comments are needed, use Chinese. # If you understand, please wait for me to give the interface definition and just answer "Understood" to save tokens. -''' +""" -ACT_PROMPT_PREFIX = '''Refer to the test types: such as missing request parameters, field boundary verification, incorrect field type. +ACT_PROMPT_PREFIX = """Refer to the test types: such as missing request parameters, field boundary verification, incorrect field type. Please output 10 test cases within one `@pytest.mark.parametrize` scope. ```text -''' +""" -YFT_PROMPT_PREFIX = '''Refer to the test types: such as SQL injection, cross-site scripting (XSS), unauthorized access and privilege escalation, +YFT_PROMPT_PREFIX = """Refer to the test types: such as SQL injection, cross-site scripting (XSS), unauthorized access and privilege escalation, authentication and authorization, parameter verification, exception handling, file upload and download. Please output 10 test cases within one `@pytest.mark.parametrize` scope. ```text -''' +""" -OCR_API_DOC = '''```text +OCR_API_DOC = """```text Interface Name: OCR recognition Interface Path: /api/v1/contract/treaty/task/ocr Method: POST @@ -96,14 +96,20 @@ def test_node_tags(project_key, nodes, operations, expected_msg): message string Yes data object Yes ``` -''' +""" class UTGenerator: """UT Generator: Construct UT through API documentation""" - def __init__(self, swagger_file: str, ut_py_path: str, questions_path: str, - chatgpt_method: str = "API", template_prefix=YFT_PROMPT_PREFIX) -> None: + def __init__( + self, + swagger_file: str, + ut_py_path: str, + questions_path: str, + chatgpt_method: str = "API", + template_prefix=YFT_PROMPT_PREFIX, + ) -> None: """Initialize UT Generator Args: @@ -274,7 +280,7 @@ def _generate_ut(self, tag, paths): def gpt_msgs_to_code(self, messages: list) -> str: """Choose based on different calling methods""" - result = '' + result = "" if self.chatgpt_method == "API": result = GPTAPI().ask_code(msgs=messages) diff --git a/metagpt/utils/file.py b/metagpt/utils/file.py index f3691549b..6bb9a1a97 100644 --- a/metagpt/utils/file.py +++ b/metagpt/utils/file.py @@ -6,9 +6,10 @@ @File : file.py @Describe : General file operations. """ -import aiofiles from pathlib import Path +import aiofiles + from metagpt.logs import logger @@ -66,10 +67,9 @@ async def read(cls, file_path: Path, chunk_size: int = None) -> bytes: if not chunk: break chunks.append(chunk) - content = b''.join(chunks) + content = b"".join(chunks) logger.debug(f"Successfully read file, the path of file: {file_path}") return content except Exception as e: logger.error(f"Error reading file: {e}") raise e - diff --git a/metagpt/utils/highlight.py b/metagpt/utils/highlight.py index e6cbb228c..2e1d6f615 100644 --- a/metagpt/utils/highlight.py +++ b/metagpt/utils/highlight.py @@ -1,22 +1,22 @@ # 添加代码语法高亮显示 from pygments import highlight as highlight_ +from pygments.formatters import HtmlFormatter, TerminalFormatter from pygments.lexers import PythonLexer, SqlLexer -from pygments.formatters import TerminalFormatter, HtmlFormatter -def highlight(code: str, language: str = 'python', formatter: str = 'terminal'): +def highlight(code: str, language: str = "python", formatter: str = "terminal"): # 指定要高亮的语言 - if language.lower() == 'python': + if language.lower() == "python": lexer = PythonLexer() - elif language.lower() == 'sql': + elif language.lower() == "sql": lexer = SqlLexer() else: raise ValueError(f"Unsupported language: {language}") # 指定输出格式 - if formatter.lower() == 'terminal': + if formatter.lower() == "terminal": formatter = TerminalFormatter() - elif formatter.lower() == 'html': + elif formatter.lower() == "html": formatter = HtmlFormatter() else: raise ValueError(f"Unsupported formatter: {formatter}") diff --git a/metagpt/utils/mmdc_ink.py b/metagpt/utils/mmdc_ink.py index 3d91cde9d..d594adb30 100644 --- a/metagpt/utils/mmdc_ink.py +++ b/metagpt/utils/mmdc_ink.py @@ -6,9 +6,9 @@ @File : mermaid.py """ import base64 -import os -from aiohttp import ClientSession,ClientError +from aiohttp import ClientError, ClientSession + from metagpt.logs import logger @@ -29,7 +29,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix): async with session.get(url) as response: if response.status == 200: text = await response.content.read() - with open(output_file, 'wb') as f: + with open(output_file, "wb") as f: f.write(text) logger.info(f"Generating {output_file}..") else: diff --git a/metagpt/utils/mmdc_playwright.py b/metagpt/utils/mmdc_playwright.py index bdbfd82ff..5d455e1c5 100644 --- a/metagpt/utils/mmdc_playwright.py +++ b/metagpt/utils/mmdc_playwright.py @@ -8,10 +8,13 @@ import os from urllib.parse import urljoin + from playwright.async_api import async_playwright + from metagpt.logs import logger -async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int: + +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """ Converts the given Mermaid code to various output formats and saves them to files. @@ -24,66 +27,72 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, Returns: int: Returns 1 if the conversion and saving were successful, -1 otherwise. """ - suffixes=['png', 'svg', 'pdf'] + suffixes = ["png", "svg", "pdf"] __dirname = os.path.dirname(os.path.abspath(__file__)) async with async_playwright() as p: browser = await p.chromium.launch() device_scale_factor = 1.0 context = await browser.new_context( - viewport={'width': width, 'height': height}, - device_scale_factor=device_scale_factor, - ) + viewport={"width": width, "height": height}, + device_scale_factor=device_scale_factor, + ) page = await context.new_page() async def console_message(msg): logger.info(msg.text) - page.on('console', console_message) + + page.on("console", console_message) try: - await page.set_viewport_size({'width': width, 'height': height}) + await page.set_viewport_size({"width": width, "height": height}) - mermaid_html_path = os.path.abspath( - os.path.join(__dirname, 'index.html')) - mermaid_html_url = urljoin('file:', mermaid_html_path) + mermaid_html_path = os.path.abspath(os.path.join(__dirname, "index.html")) + mermaid_html_url = urljoin("file:", mermaid_html_path) await page.goto(mermaid_html_url) await page.wait_for_load_state("networkidle") await page.wait_for_selector("div#container", state="attached") - mermaid_config = {} + # mermaid_config = {} background_color = "#ffffff" - my_css = "" + # my_css = "" await page.evaluate(f'document.body.style.background = "{background_color}";') - metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => { - const { mermaid, zenuml } = globalThis; - await mermaid.registerExternalDiagrams([zenuml]); - mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); - const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); - document.getElementById('container').innerHTML = svg; - const svgElement = document.querySelector('svg'); - svgElement.style.backgroundColor = backgroundColor; - - if (myCSS) { - const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); - style.appendChild(document.createTextNode(myCSS)); - svgElement.appendChild(style); - } - - }''', [mermaid_code, mermaid_config, my_css, background_color]) - - if 'svg' in suffixes : - svg_xml = await page.evaluate('''() => { + # metadata = await page.evaluate( + # """async ([definition, mermaidConfig, myCSS, backgroundColor]) => { + # const { mermaid, zenuml } = globalThis; + # await mermaid.registerExternalDiagrams([zenuml]); + # mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); + # const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); + # document.getElementById('container').innerHTML = svg; + # const svgElement = document.querySelector('svg'); + # svgElement.style.backgroundColor = backgroundColor; + # + # if (myCSS) { + # const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); + # style.appendChild(document.createTextNode(myCSS)); + # svgElement.appendChild(style); + # } + # + # }""", + # [mermaid_code, mermaid_config, my_css, background_color], + # ) + + if "svg" in suffixes: + svg_xml = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const xmlSerializer = new XMLSerializer(); return xmlSerializer.serializeToString(svg); - }''') + }""" + ) logger.info(f"Generating {output_file_without_suffix}.svg..") - with open(f'{output_file_without_suffix}.svg', 'wb') as f: - f.write(svg_xml.encode('utf-8')) + with open(f"{output_file_without_suffix}.svg", "wb") as f: + f.write(svg_xml.encode("utf-8")) - if 'png' in suffixes: - clip = await page.evaluate('''() => { + if "png" in suffixes: + clip = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const rect = svg.getBoundingClientRect(); return { @@ -92,16 +101,17 @@ async def console_message(msg): width: Math.ceil(rect.width), height: Math.ceil(rect.height) }; - }''') - await page.set_viewport_size({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height']}) - screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + }""" + ) + await page.set_viewport_size({"width": clip["x"] + clip["width"], "height": clip["y"] + clip["height"]}) + screenshot = await page.screenshot(clip=clip, omit_background=True, scale="device") logger.info(f"Generating {output_file_without_suffix}.png..") - with open(f'{output_file_without_suffix}.png', 'wb') as f: + with open(f"{output_file_without_suffix}.png", "wb") as f: f.write(screenshot) - if 'pdf' in suffixes: + if "pdf" in suffixes: pdf_data = await page.pdf(scale=device_scale_factor) logger.info(f"Generating {output_file_without_suffix}.pdf..") - with open(f'{output_file_without_suffix}.pdf', 'wb') as f: + with open(f"{output_file_without_suffix}.pdf", "wb") as f: f.write(pdf_data) return 0 except Exception as e: diff --git a/metagpt/utils/mmdc_pyppeteer.py b/metagpt/utils/mmdc_pyppeteer.py index 7ec30fd12..7125cafc5 100644 --- a/metagpt/utils/mmdc_pyppeteer.py +++ b/metagpt/utils/mmdc_pyppeteer.py @@ -7,11 +7,14 @@ """ import os from urllib.parse import urljoin + from pyppeteer import launch -from metagpt.logs import logger + from metagpt.config import CONFIG +from metagpt.logs import logger -async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int: + +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """ Converts the given Mermaid code to various output formats and saves them to files. @@ -24,15 +27,15 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, Returns: int: Returns 1 if the conversion and saving were successful, -1 otherwise. """ - suffixes = ['png', 'svg', 'pdf'] + suffixes = ["png", "svg", "pdf"] __dirname = os.path.dirname(os.path.abspath(__file__)) - if CONFIG.pyppeteer_executable_path: - browser = await launch(headless=True, - executablePath=CONFIG.pyppeteer_executable_path, - args=['--disable-extensions',"--no-sandbox"] - ) + browser = await launch( + headless=True, + executablePath=CONFIG.pyppeteer_executable_path, + args=["--disable-extensions", "--no-sandbox"], + ) else: logger.error("Please set the environment variable:PYPPETEER_EXECUTABLE_PATH.") return -1 @@ -41,50 +44,56 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, async def console_message(msg): logger.info(msg.text) - page.on('console', console_message) + + page.on("console", console_message) try: - await page.setViewport(viewport={'width': width, 'height': height, 'deviceScaleFactor': device_scale_factor}) + await page.setViewport(viewport={"width": width, "height": height, "deviceScaleFactor": device_scale_factor}) - mermaid_html_path = os.path.abspath( - os.path.join(__dirname, 'index.html')) - mermaid_html_url = urljoin('file:', mermaid_html_path) + mermaid_html_path = os.path.abspath(os.path.join(__dirname, "index.html")) + mermaid_html_url = urljoin("file:", mermaid_html_path) await page.goto(mermaid_html_url) await page.querySelector("div#container") - mermaid_config = {} + # mermaid_config = {} background_color = "#ffffff" - my_css = "" + # my_css = "" await page.evaluate(f'document.body.style.background = "{background_color}";') - metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => { - const { mermaid, zenuml } = globalThis; - await mermaid.registerExternalDiagrams([zenuml]); - mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); - const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); - document.getElementById('container').innerHTML = svg; - const svgElement = document.querySelector('svg'); - svgElement.style.backgroundColor = backgroundColor; + # metadata = await page.evaluate( + # """async ([definition, mermaidConfig, myCSS, backgroundColor]) => { + # const { mermaid, zenuml } = globalThis; + # await mermaid.registerExternalDiagrams([zenuml]); + # mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); + # const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); + # document.getElementById('container').innerHTML = svg; + # const svgElement = document.querySelector('svg'); + # svgElement.style.backgroundColor = backgroundColor; + # + # if (myCSS) { + # const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); + # style.appendChild(document.createTextNode(myCSS)); + # svgElement.appendChild(style); + # } + # }""", + # [mermaid_code, mermaid_config, my_css, background_color], + # ) - if (myCSS) { - const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); - style.appendChild(document.createTextNode(myCSS)); - svgElement.appendChild(style); - } - }''', [mermaid_code, mermaid_config, my_css, background_color]) - - if 'svg' in suffixes : - svg_xml = await page.evaluate('''() => { + if "svg" in suffixes: + svg_xml = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const xmlSerializer = new XMLSerializer(); return xmlSerializer.serializeToString(svg); - }''') + }""" + ) logger.info(f"Generating {output_file_without_suffix}.svg..") - with open(f'{output_file_without_suffix}.svg', 'wb') as f: - f.write(svg_xml.encode('utf-8')) + with open(f"{output_file_without_suffix}.svg", "wb") as f: + f.write(svg_xml.encode("utf-8")) - if 'png' in suffixes: - clip = await page.evaluate('''() => { + if "png" in suffixes: + clip = await page.evaluate( + """() => { const svg = document.querySelector('svg'); const rect = svg.getBoundingClientRect(); return { @@ -93,16 +102,23 @@ async def console_message(msg): width: Math.ceil(rect.width), height: Math.ceil(rect.height) }; - }''') - await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'], 'deviceScaleFactor': device_scale_factor}) - screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + }""" + ) + await page.setViewport( + { + "width": clip["x"] + clip["width"], + "height": clip["y"] + clip["height"], + "deviceScaleFactor": device_scale_factor, + } + ) + screenshot = await page.screenshot(clip=clip, omit_background=True, scale="device") logger.info(f"Generating {output_file_without_suffix}.png..") - with open(f'{output_file_without_suffix}.png', 'wb') as f: + with open(f"{output_file_without_suffix}.png", "wb") as f: f.write(screenshot) - if 'pdf' in suffixes: + if "pdf" in suffixes: pdf_data = await page.pdf(scale=device_scale_factor) logger.info(f"Generating {output_file_without_suffix}.pdf..") - with open(f'{output_file_without_suffix}.pdf', 'wb') as f: + with open(f"{output_file_without_suffix}.pdf", "wb") as f: f.write(pdf_data) return 0 except Exception as e: @@ -110,4 +126,3 @@ async def console_message(msg): return -1 finally: await browser.close() - diff --git a/metagpt/utils/parse_html.py b/metagpt/utils/parse_html.py index 62de26541..f2395026f 100644 --- a/metagpt/utils/parse_html.py +++ b/metagpt/utils/parse_html.py @@ -16,7 +16,7 @@ class WebPage(BaseModel): class Config: underscore_attrs_are_private = True - _soup : Optional[BeautifulSoup] = None + _soup: Optional[BeautifulSoup] = None _title: Optional[str] = None @property @@ -24,7 +24,7 @@ def soup(self) -> BeautifulSoup: if self._soup is None: self._soup = BeautifulSoup(self.html, "html.parser") return self._soup - + @property def title(self): if self._title is None: diff --git a/metagpt/utils/pycst.py b/metagpt/utils/pycst.py index afd85a547..1edfed81c 100644 --- a/metagpt/utils/pycst.py +++ b/metagpt/utils/pycst.py @@ -37,12 +37,12 @@ def get_docstring_statement(body: DocstringNode) -> cst.SimpleStatementLine: if not isinstance(expr, cst.Expr): return None - + val = expr.value if not isinstance(val, (cst.SimpleString, cst.ConcatenatedString)): return None - - evaluated_value = val.evaluated_value + + evaluated_value = val.evaluated_value if isinstance(evaluated_value, bytes): return None @@ -56,6 +56,7 @@ class DocstringCollector(cst.CSTVisitor): stack: A list to keep track of the current path in the CST. docstrings: A dictionary mapping paths in the CST to their corresponding docstrings. """ + def __init__(self): self.stack: list[str] = [] self.docstrings: dict[tuple[str, ...], cst.SimpleStatementLine] = {} @@ -96,6 +97,7 @@ class DocstringTransformer(cst.CSTTransformer): stack: A list to keep track of the current path in the CST. docstrings: A dictionary mapping paths in the CST to their corresponding docstrings. """ + def __init__( self, docstrings: dict[tuple[str, ...], cst.SimpleStatementLine], @@ -125,7 +127,9 @@ def _leave(self, original_node: DocstringNode, updated_node: DocstringNode) -> D key = tuple(self.stack) self.stack.pop() - if hasattr(updated_node, "decorators") and any((i.decorator.value == "overload") for i in updated_node.decorators): + if hasattr(updated_node, "decorators") and any( + (i.decorator.value == "overload") for i in updated_node.decorators + ): return updated_node statement = self.docstrings.get(key) diff --git a/metagpt/utils/read_document.py b/metagpt/utils/read_document.py index c837baf25..d2fafbc17 100644 --- a/metagpt/utils/read_document.py +++ b/metagpt/utils/read_document.py @@ -8,6 +8,7 @@ import docx + def read_docx(file_path: str) -> list: """Open a docx file""" doc = docx.Document(file_path) diff --git a/metagpt/utils/singleton.py b/metagpt/utils/singleton.py index 474b537db..a9e0862c0 100644 --- a/metagpt/utils/singleton.py +++ b/metagpt/utils/singleton.py @@ -20,4 +20,3 @@ def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] - \ No newline at end of file diff --git a/metagpt/utils/special_tokens.py b/metagpt/utils/special_tokens.py index 2adb93c77..5e780ce05 100644 --- a/metagpt/utils/special_tokens.py +++ b/metagpt/utils/special_tokens.py @@ -1,4 +1,4 @@ # token to separate different code messages in a WriteCode Message content -MSG_SEP = "#*000*#" +MSG_SEP = "#*000*#" # token to seperate file name and the actual code text in a code message FILENAME_CODE_SEP = "#*001*#" diff --git a/metagpt/utils/text.py b/metagpt/utils/text.py index be3c52edd..dd9678438 100644 --- a/metagpt/utils/text.py +++ b/metagpt/utils/text.py @@ -3,7 +3,12 @@ from metagpt.utils.token_counter import TOKEN_MAX, count_string_tokens -def reduce_message_length(msgs: Generator[str, None, None], model_name: str, system_text: str, reserved: int = 0,) -> str: +def reduce_message_length( + msgs: Generator[str, None, None], + model_name: str, + system_text: str, + reserved: int = 0, +) -> str: """Reduce the length of concatenated message segments to fit within the maximum token size. Args: @@ -49,9 +54,9 @@ def generate_prompt_chunk( current_token = 0 current_lines = [] - reserved = reserved + count_string_tokens(prompt_template+system_text, model_name) + reserved = reserved + count_string_tokens(prompt_template + system_text, model_name) # 100 is a magic number to ensure the maximum context length is not exceeded - max_token = TOKEN_MAX.get(model_name, 2048) - reserved - 100 + max_token = TOKEN_MAX.get(model_name, 2048) - reserved - 100 while paragraphs: paragraph = paragraphs.pop(0) @@ -103,7 +108,7 @@ def decode_unicode_escape(text: str) -> str: return text.encode("utf-8").decode("unicode_escape", "ignore") -def _split_by_count(lst: Sequence , count: int): +def _split_by_count(lst: Sequence, count: int): avg = len(lst) // count remainder = len(lst) % count start = 0 diff --git a/tests/conftest.py b/tests/conftest.py index feecc7715..d2ac8304f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,14 +6,14 @@ @File : conftest.py """ +import asyncio +import re from unittest.mock import Mock import pytest from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI -import asyncio -import re class Context: diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index a800690e8..23d10ccc4 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -311,12 +311,10 @@ class Game{ "添加数据API：接受用户输入的文档库，对文档库进行索引\n- 使用MeiliSearch连接并添加文档库", "搜索API：接收用户输入的关键词，返回相关的搜索结果\n- 使用MeiliSearch连接并使用接口获得对应数据", "多条件筛选API：接收用户选择的筛选条件，返回符合条件的搜索结果。\n- 使用MeiliSearch进行筛选并返回符合条件的搜索结果", - "智能推荐API：根据用户的搜索历史记录和搜索行为，推荐相关的搜索结果。" + "智能推荐API：根据用户的搜索历史记录和搜索行为，推荐相关的搜索结果。", ] -TASKS_2 = [ - "完成main.py的功能" -] +TASKS_2 = ["完成main.py的功能"] SEARCH_CODE_SAMPLE = """ import requests @@ -460,7 +458,7 @@ def format_results(self, search_results): print('No results found.') ''' -MEILI_CODE = '''import meilisearch +MEILI_CODE = """import meilisearch from typing import List @@ -496,9 +494,9 @@ def add_documents(self, data_source: DataSource, documents: List[dict]): # 添加文档库到搜索引擎 search_engine.add_documents(books_data_source, documents) -''' +""" -MEILI_ERROR = '''/usr/local/bin/python3.9 /Users/alexanderwu/git/metagpt/examples/search/meilisearch_index.py +MEILI_ERROR = """/usr/local/bin/python3.9 /Users/alexanderwu/git/metagpt/examples/search/meilisearch_index.py Traceback (most recent call last): File "/Users/alexanderwu/git/metagpt/examples/search/meilisearch_index.py", line 44, in search_engine.add_documents(books_data_source, documents) @@ -506,7 +504,7 @@ def add_documents(self, data_source: DataSource, documents: List[dict]): index = self.client.get_or_create_index(index_name) AttributeError: 'Client' object has no attribute 'get_or_create_index' -Process finished with exit code 1''' +Process finished with exit code 1""" MEILI_CODE_REFINED = """ """ diff --git a/tests/metagpt/actions/test_action_output.py b/tests/metagpt/actions/test_action_output.py index a556789db..ef8e239bd 100644 --- a/tests/metagpt/actions/test_action_output.py +++ b/tests/metagpt/actions/test_action_output.py @@ -9,18 +9,21 @@ from metagpt.actions import ActionOutput -t_dict = {"Required Python third-party packages": "\"\"\"\nflask==1.1.2\npygame==2.0.1\n\"\"\"\n", - "Required Other language third-party packages": "\"\"\"\nNo third-party packages required for other languages.\n\"\"\"\n", - "Full API spec": "\"\"\"\nopenapi: 3.0.0\ninfo:\n title: Web Snake Game API\n version: 1.0.0\npaths:\n /game:\n get:\n summary: Get the current game state\n responses:\n '200':\n description: A JSON object of the game state\n post:\n summary: Send a command to the game\n requestBody:\n required: true\n content:\n application/json:\n schema:\n type: object\n properties:\n command:\n type: string\n responses:\n '200':\n description: A JSON object of the updated game state\n\"\"\"\n", - "Logic Analysis": [ - ["app.py", "Main entry point for the Flask application. Handles HTTP requests and responses."], - ["game.py", "Contains the Game and Snake classes. Handles the game logic."], - ["static/js/script.js", "Handles user interactions and updates the game UI."], - ["static/css/styles.css", "Defines the styles for the game UI."], - ["templates/index.html", "The main page of the web application. Displays the game UI."]], - "Task list": ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"], - "Shared Knowledge": "\"\"\"\n'game.py' contains the Game and Snake classes which are responsible for the game logic. The Game class uses an instance of the Snake class.\n\n'app.py' is the main entry point for the Flask application. It creates an instance of the Game class and handles HTTP requests and responses.\n\n'static/js/script.js' is responsible for handling user interactions and updating the game UI based on the game state returned by 'app.py'.\n\n'static/css/styles.css' defines the styles for the game UI.\n\n'templates/index.html' is the main page of the web application. It displays the game UI and loads 'static/js/script.js' and 'static/css/styles.css'.\n\"\"\"\n", - "Anything UNCLEAR": "We need clarification on how the high score should be stored. Should it persist across sessions (stored in a database or a file) or should it reset every time the game is restarted? Also, should the game speed increase as the snake grows, or should it remain constant throughout the game?"} +t_dict = { + "Required Python third-party packages": '"""\nflask==1.1.2\npygame==2.0.1\n"""\n', + "Required Other language third-party packages": '"""\nNo third-party packages required for other languages.\n"""\n', + "Full API spec": '"""\nopenapi: 3.0.0\ninfo:\n title: Web Snake Game API\n version: 1.0.0\npaths:\n /game:\n get:\n summary: Get the current game state\n responses:\n \'200\':\n description: A JSON object of the game state\n post:\n summary: Send a command to the game\n requestBody:\n required: true\n content:\n application/json:\n schema:\n type: object\n properties:\n command:\n type: string\n responses:\n \'200\':\n description: A JSON object of the updated game state\n"""\n', + "Logic Analysis": [ + ["app.py", "Main entry point for the Flask application. Handles HTTP requests and responses."], + ["game.py", "Contains the Game and Snake classes. Handles the game logic."], + ["static/js/script.js", "Handles user interactions and updates the game UI."], + ["static/css/styles.css", "Defines the styles for the game UI."], + ["templates/index.html", "The main page of the web application. Displays the game UI."], + ], + "Task list": ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"], + "Shared Knowledge": "\"\"\"\n'game.py' contains the Game and Snake classes which are responsible for the game logic. The Game class uses an instance of the Snake class.\n\n'app.py' is the main entry point for the Flask application. It creates an instance of the Game class and handles HTTP requests and responses.\n\n'static/js/script.js' is responsible for handling user interactions and updating the game UI based on the game state returned by 'app.py'.\n\n'static/css/styles.css' defines the styles for the game UI.\n\n'templates/index.html' is the main page of the web application. It displays the game UI and loads 'static/js/script.js' and 'static/css/styles.css'.\n\"\"\"\n", + "Anything UNCLEAR": "We need clarification on how the high score should be stored. Should it persist across sessions (stored in a database or a file) or should it reset every time the game is restarted? Also, should the game speed increase as the snake grows, or should it remain constant throughout the game?", +} WRITE_TASKS_OUTPUT_MAPPING = { "Required Python third-party packages": (str, ...), @@ -45,6 +48,6 @@ def test_create_model_class_with_mapping(): assert value == ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"] -if __name__ == '__main__': +if __name__ == "__main__": test_create_model_class() test_create_model_class_with_mapping() diff --git a/tests/metagpt/actions/test_azure_tts.py b/tests/metagpt/actions/test_azure_tts.py index b5a333af2..bcafe10f5 100644 --- a/tests/metagpt/actions/test_azure_tts.py +++ b/tests/metagpt/actions/test_azure_tts.py @@ -10,12 +10,7 @@ def test_azure_tts(): azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( - "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") + azure_tts.synthesize_speech("zh-CN", "zh-CN-YunxiNeural", "Boy", "你好，我是卡卡", "output.wav") # 运行需要先配置 SUBSCRIPTION_KEY # TODO: 这里如果要检验，还要额外加上对应的asr，才能确保前后生成是接近一致的，但现在还没有 diff --git a/tests/metagpt/actions/test_clone_function.py b/tests/metagpt/actions/test_clone_function.py index 6d4432dcd..44248eb80 100644 --- a/tests/metagpt/actions/test_clone_function.py +++ b/tests/metagpt/actions/test_clone_function.py @@ -2,7 +2,6 @@ from metagpt.actions.clone_function import CloneFunction, run_function_code - source_code = """ import pandas as pd import ta @@ -31,14 +30,18 @@ def get_expected_res(): import ta # 读取股票数据 - stock_data = pd.read_csv('./tests/data/baba_stock.csv') + stock_data = pd.read_csv("./tests/data/baba_stock.csv") stock_data.head() # 计算简单移动平均线 - stock_data['SMA'] = ta.trend.sma_indicator(stock_data['Close'], window=6) - stock_data[['Date', 'Close', 'SMA']].head() + stock_data["SMA"] = ta.trend.sma_indicator(stock_data["Close"], window=6) + stock_data[["Date", "Close", "SMA"]].head() # 计算布林带 - stock_data['bb_upper'], stock_data['bb_middle'], stock_data['bb_lower'] = ta.volatility.bollinger_hband_indicator(stock_data['Close'], window=20), ta.volatility.bollinger_mavg(stock_data['Close'], window=20), ta.volatility.bollinger_lband_indicator(stock_data['Close'], window=20) - stock_data[['Date', 'Close', 'bb_upper', 'bb_middle', 'bb_lower']].head() + stock_data["bb_upper"], stock_data["bb_middle"], stock_data["bb_lower"] = ( + ta.volatility.bollinger_hband_indicator(stock_data["Close"], window=20), + ta.volatility.bollinger_mavg(stock_data["Close"], window=20), + ta.volatility.bollinger_lband_indicator(stock_data["Close"], window=20), + ) + stock_data[["Date", "Close", "bb_upper", "bb_middle", "bb_lower"]].head() return stock_data @@ -46,9 +49,9 @@ def get_expected_res(): async def test_clone_function(): clone = CloneFunction() code = await clone.run(template_code, source_code) - assert 'def ' in code - stock_path = './tests/data/baba_stock.csv' - df, msg = run_function_code(code, 'stock_indicator', stock_path) + assert "def " in code + stock_path = "./tests/data/baba_stock.csv" + df, msg = run_function_code(code, "stock_indicator", stock_path) assert not msg expected_df = get_expected_res() assert df.equals(expected_df) diff --git a/tests/metagpt/actions/test_debug_error.py b/tests/metagpt/actions/test_debug_error.py index 555c84e4e..2393d2cc9 100644 --- a/tests/metagpt/actions/test_debug_error.py +++ b/tests/metagpt/actions/test_debug_error.py @@ -144,12 +144,12 @@ def test_player_calculate_score_with_multiple_aces(self): --- ''' + @pytest.mark.asyncio async def test_debug_error(): - debug_error = DebugError("debug_error") file_name, rewritten_code = await debug_error.run(context=EXAMPLE_MSG_CONTENT) - assert "class Player" in rewritten_code # rewrite the same class - assert "while self.score > 21" in rewritten_code # a key logic to rewrite to (original one is "if self.score > 12") + assert "class Player" in rewritten_code # rewrite the same class + assert "while self.score > 21" in rewritten_code # a key logic to rewrite to (original one is "if self.score > 12") diff --git a/tests/metagpt/actions/test_detail_mining.py b/tests/metagpt/actions/test_detail_mining.py index c9d5331f9..891dca6ca 100644 --- a/tests/metagpt/actions/test_detail_mining.py +++ b/tests/metagpt/actions/test_detail_mining.py @@ -10,6 +10,7 @@ from metagpt.actions.detail_mining import DetailMining from metagpt.logs import logger + @pytest.mark.asyncio async def test_detail_mining(): topic = "如何做一个生日蛋糕" @@ -17,7 +18,6 @@ async def test_detail_mining(): detail_mining = DetailMining("detail_mining") rsp = await detail_mining.run(topic=topic, record=record) logger.info(f"{rsp.content=}") - - assert '##OUTPUT' in rsp.content - assert '蛋糕' in rsp.content + assert "##OUTPUT" in rsp.content + assert "蛋糕" in rsp.content diff --git a/tests/metagpt/actions/test_invoice_ocr.py b/tests/metagpt/actions/test_invoice_ocr.py index a15166f7c..7f16aa9a4 100644 --- a/tests/metagpt/actions/test_invoice_ocr.py +++ b/tests/metagpt/actions/test_invoice_ocr.py @@ -8,12 +8,11 @@ """ import os -from typing import List +from pathlib import Path import pytest -from pathlib import Path -from metagpt.actions.invoice_ocr import InvoiceOCR, GenerateTable, ReplyQuestion +from metagpt.actions.invoice_ocr import GenerateTable, InvoiceOCR, ReplyQuestion @pytest.mark.asyncio @@ -22,7 +21,7 @@ [ "../../data/invoices/invoice-3.jpg", "../../data/invoices/invoice-4.zip", - ] + ], ) async def test_invoice_ocr(invoice_path: str): invoice_path = os.path.abspath(os.path.join(os.getcwd(), invoice_path)) @@ -35,18 +34,8 @@ async def test_invoice_ocr(invoice_path: str): @pytest.mark.parametrize( ("invoice_path", "expected_result"), [ - ( - "../../data/invoices/invoice-1.pdf", - [ - { - "收款人": "小明", - "城市": "深圳市", - "总费用/元": "412.00", - "开票日期": "2023年02月03日" - } - ] - ), - ] + ("../../data/invoices/invoice-1.pdf", [{"收款人": "小明", "城市": "深圳市", "总费用/元": "412.00", "开票日期": "2023年02月03日"}]), + ], ) async def test_generate_table(invoice_path: str, expected_result: list[dict]): invoice_path = os.path.abspath(os.path.join(os.getcwd(), invoice_path)) @@ -59,9 +48,7 @@ async def test_generate_table(invoice_path: str, expected_result: list[dict]): @pytest.mark.asyncio @pytest.mark.parametrize( ("invoice_path", "query", "expected_result"), - [ - ("../../data/invoices/invoice-1.pdf", "Invoicing date", "2023年02月03日") - ] + [("../../data/invoices/invoice-1.pdf", "Invoicing date", "2023年02月03日")], ) async def test_reply_question(invoice_path: str, query: dict, expected_result: str): invoice_path = os.path.abspath(os.path.join(os.getcwd(), invoice_path)) @@ -69,4 +56,3 @@ async def test_reply_question(invoice_path: str, query: dict, expected_result: s ocr_result = await InvoiceOCR().run(file_path=Path(invoice_path), filename=filename) result = await ReplyQuestion().run(query=query, ocr_result=ocr_result) assert expected_result in result - diff --git a/tests/metagpt/actions/test_ui_design.py b/tests/metagpt/actions/test_ui_design.py index d284b20f2..b8be914ae 100644 --- a/tests/metagpt/actions/test_ui_design.py +++ b/tests/metagpt/actions/test_ui_design.py @@ -4,7 +4,7 @@ # from tests.metagpt.roles.ui_role import UIDesign -llm_resp= ''' +llm_resp = """ # UI Design Description ```The user interface for the snake game will be designed in a way that is simple, clean, and intuitive. The main elements of the game such as the game grid, snake, food, score, and game over message will be clearly defined and easy to understand. The game grid will be centered on the screen with the score displayed at the top. The game controls will be intuitive and easy to use. The design will be modern and minimalist with a pleasing color scheme.``` @@ -98,12 +98,13 @@ left: 50%; transform: translate(-50%, -50%); font-size: 3em; - ''' + """ + def test_ui_design_parse_css(): ui_design_work = UIDesign(name="UI design action") - css = ''' + css = """ body { display: flex; flex-direction: column; @@ -160,14 +161,14 @@ def test_ui_design_parse_css(): left: 50%; transform: translate(-50%, -50%); font-size: 3em; - ''' - assert ui_design_work.parse_css_code(context=llm_resp)==css + """ + assert ui_design_work.parse_css_code(context=llm_resp) == css def test_ui_design_parse_html(): ui_design_work = UIDesign(name="UI design action") - html = ''' + html = """ @@ -184,8 +185,5 @@ def test_ui_design_parse_html():
Game Over
- ''' - assert ui_design_work.parse_css_code(context=llm_resp)==html - - - + """ + assert ui_design_work.parse_css_code(context=llm_resp) == html diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 7bb18ddf2..eb5e3de91 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -22,13 +22,13 @@ async def test_write_code(): logger.info(code) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 - assert 'def add' in code - assert 'return' in code + assert "def add" in code + assert "return" in code @pytest.mark.asyncio async def test_write_code_directly(): - prompt = WRITE_CODE_PROMPT_SAMPLE + '\n' + TASKS_2[0] + prompt = WRITE_CODE_PROMPT_SAMPLE + "\n" + TASKS_2[0] llm = LLM() rsp = await llm.aask(prompt) logger.info(rsp) diff --git a/tests/metagpt/actions/test_write_docstring.py b/tests/metagpt/actions/test_write_docstring.py index 82d96e1a6..a8a80b36d 100644 --- a/tests/metagpt/actions/test_write_docstring.py +++ b/tests/metagpt/actions/test_write_docstring.py @@ -2,7 +2,7 @@ from metagpt.actions.write_docstring import WriteDocstring -code = ''' +code = """ def add_numbers(a: int, b: int): return a + b @@ -14,7 +14,7 @@ def __init__(self, name: str, age: int): def greet(self): return f"Hello, my name is {self.name} and I am {self.age} years old." -''' +""" @pytest.mark.asyncio @@ -25,7 +25,7 @@ def greet(self): ("numpy", "Parameters"), ("sphinx", ":param name:"), ], - ids=["google", "numpy", "sphinx"] + ids=["google", "numpy", "sphinx"], ) async def test_write_docstring(style: str, part: str): ret = await WriteDocstring().run(code, style=style) diff --git a/tests/metagpt/actions/test_write_tutorial.py b/tests/metagpt/actions/test_write_tutorial.py index 683fee082..27a323b44 100644 --- a/tests/metagpt/actions/test_write_tutorial.py +++ b/tests/metagpt/actions/test_write_tutorial.py @@ -9,14 +9,11 @@ import pytest -from metagpt.actions.write_tutorial import WriteDirectory, WriteContent +from metagpt.actions.write_tutorial import WriteContent, WriteDirectory @pytest.mark.asyncio -@pytest.mark.parametrize( - ("language", "topic"), - [("English", "Write a tutorial about Python")] -) +@pytest.mark.parametrize(("language", "topic"), [("English", "Write a tutorial about Python")]) async def test_write_directory(language: str, topic: str): ret = await WriteDirectory(language=language).run(topic=topic) assert isinstance(ret, dict) @@ -30,7 +27,7 @@ async def test_write_directory(language: str, topic: str): @pytest.mark.asyncio @pytest.mark.parametrize( ("language", "topic", "directory"), - [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})] + [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})], ) async def test_write_content(language: str, topic: str, directory: Dict): ret = await WriteContent(language=language, directory=directory).run(topic=topic) diff --git a/tests/metagpt/document_store/test_chromadb_store.py b/tests/metagpt/document_store/test_chromadb_store.py index f8c11e1ca..fd115dcdd 100644 --- a/tests/metagpt/document_store/test_chromadb_store.py +++ b/tests/metagpt/document_store/test_chromadb_store.py @@ -12,12 +12,12 @@ def test_chroma_store(): """FIXME：chroma使用感觉很诡异，一用Python就挂，测试用例里也是""" # 创建 ChromaStore 实例，使用 'sample_collection' 集合 - document_store = ChromaStore('sample_collection_1') + document_store = ChromaStore("sample_collection_1") # 使用 write 方法添加多个文档 - document_store.write(["This is document1", "This is document2"], - [{"source": "google-docs"}, {"source": "notion"}], - ["doc1", "doc2"]) + document_store.write( + ["This is document1", "This is document2"], [{"source": "google-docs"}, {"source": "notion"}], ["doc1", "doc2"] + ) # 使用 add 方法添加一个文档 document_store.add("This is document3", {"source": "notion"}, "doc3") diff --git a/tests/metagpt/document_store/test_faiss_store.py b/tests/metagpt/document_store/test_faiss_store.py index d22d234f5..f14bee817 100644 --- a/tests/metagpt/document_store/test_faiss_store.py +++ b/tests/metagpt/document_store/test_faiss_store.py @@ -39,11 +39,11 @@ @pytest.mark.asyncio async def test_faiss_store_search(): - store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json') - store.add(['油皮洗面奶']) + store = FaissStore(DATA_PATH / "qcs/qcs_4w.json") + store.add(["油皮洗面奶"]) role = Sales(store=store) - queries = ['油皮洗面奶', '介绍下欧莱雅的'] + queries = ["油皮洗面奶", "介绍下欧莱雅的"] for query in queries: rsp = await role.run(query) assert rsp @@ -60,7 +60,10 @@ def customer_service(): async def test_faiss_store_customer_service(): allq = [ # ["我的餐怎么两小时都没到", "退货吧"], - ["你好收不到取餐码，麻烦帮我开箱", "14750187158", ] + [ + "你好收不到取餐码，麻烦帮我开箱", + "14750187158", + ] ] role = customer_service() for queries in allq: @@ -71,4 +74,4 @@ async def test_faiss_store_customer_service(): def test_faiss_store_no_file(): with pytest.raises(FileNotFoundError): - FaissStore(DATA_PATH / 'wtf.json') + FaissStore(DATA_PATH / "wtf.json") diff --git a/tests/metagpt/document_store/test_lancedb_store.py b/tests/metagpt/document_store/test_lancedb_store.py index 9c2f9fb42..5c0e40f57 100644 --- a/tests/metagpt/document_store/test_lancedb_store.py +++ b/tests/metagpt/document_store/test_lancedb_store.py @@ -5,27 +5,33 @@ @Author : unkn-wn (Leon Yee) @File : test_lancedb_store.py """ -from metagpt.document_store.lancedb_store import LanceStore -import pytest import random +import pytest + +from metagpt.document_store.lancedb_store import LanceStore + + @pytest def test_lance_store(): - # This simply establishes the connection to the database, so we can drop the table if it exists - store = LanceStore('test') + store = LanceStore("test") - store.drop('test') + store.drop("test") - store.write(data=[[random.random() for _ in range(100)] for _ in range(2)], - metadatas=[{"source": "google-docs"}, {"source": "notion"}], - ids=["doc1", "doc2"]) + store.write( + data=[[random.random() for _ in range(100)] for _ in range(2)], + metadatas=[{"source": "google-docs"}, {"source": "notion"}], + ids=["doc1", "doc2"], + ) store.add(data=[random.random() for _ in range(100)], metadata={"source": "notion"}, _id="doc3") result = store.search([random.random() for _ in range(100)], n_results=3) - assert(len(result) == 3) + assert len(result) == 3 store.delete("doc2") - result = store.search([random.random() for _ in range(100)], n_results=3, where="source = 'notion'", metric='cosine') - assert(len(result) == 1) \ No newline at end of file + result = store.search( + [random.random() for _ in range(100)], n_results=3, where="source = 'notion'", metric="cosine" + ) + assert len(result) == 1 diff --git a/tests/metagpt/document_store/test_milvus_store.py b/tests/metagpt/document_store/test_milvus_store.py index 1cf65776d..34497b9c6 100644 --- a/tests/metagpt/document_store/test_milvus_store.py +++ b/tests/metagpt/document_store/test_milvus_store.py @@ -12,7 +12,7 @@ from metagpt.document_store.milvus_store import MilvusConnection, MilvusStore from metagpt.logs import logger -book_columns = {'idx': int, 'name': str, 'desc': str, 'emb': np.ndarray, 'price': float} +book_columns = {"idx": int, "name": str, "desc": str, "emb": np.ndarray, "price": float} book_data = [ [i for i in range(10)], [f"book-{i}" for i in range(10)], @@ -25,12 +25,12 @@ def test_milvus_store(): milvus_connection = MilvusConnection(alias="default", host="192.168.50.161", port="30530") milvus_store = MilvusStore(milvus_connection) - milvus_store.drop('Book') - milvus_store.create_collection('Book', book_columns) + milvus_store.drop("Book") + milvus_store.create_collection("Book", book_columns) milvus_store.add(book_data) - milvus_store.build_index('emb') + milvus_store.build_index("emb") milvus_store.load_collection() - results = milvus_store.search([[1.0, 1.0]], field='emb') + results = milvus_store.search([[1.0, 1.0]], field="emb") logger.info(results) assert results diff --git a/tests/metagpt/document_store/test_qdrant_store.py b/tests/metagpt/document_store/test_qdrant_store.py index a63a4329d..cdd619d37 100644 --- a/tests/metagpt/document_store/test_qdrant_store.py +++ b/tests/metagpt/document_store/test_qdrant_store.py @@ -24,9 +24,7 @@ vectors = [[random.random() for _ in range(2)] for _ in range(10)] points = [ - PointStruct( - id=idx, vector=vector, payload={"color": "red", "rand_number": idx % 10} - ) + PointStruct(id=idx, vector=vector, payload={"color": "red", "rand_number": idx % 10}) for idx, vector in enumerate(vectors) ] @@ -57,9 +55,7 @@ def test_milvus_store(): results = qdrant_store.search( "Book", query=[1.0, 1.0], - query_filter=Filter( - must=[FieldCondition(key="rand_number", range=Range(gte=8))] - ), + query_filter=Filter(must=[FieldCondition(key="rand_number", range=Range(gte=8))]), ) assert results[0]["id"] == 8 assert results[0]["score"] == 0.9100373450784073 @@ -68,9 +64,7 @@ def test_milvus_store(): results = qdrant_store.search( "Book", query=[1.0, 1.0], - query_filter=Filter( - must=[FieldCondition(key="rand_number", range=Range(gte=8))] - ), + query_filter=Filter(must=[FieldCondition(key="rand_number", range=Range(gte=8))]), return_vector=True, ) assert results[0]["vector"] == [0.35037919878959656, 0.9366079568862915] diff --git a/tests/metagpt/management/test_skill_manager.py b/tests/metagpt/management/test_skill_manager.py index b0be858a1..462bc23a6 100644 --- a/tests/metagpt/management/test_skill_manager.py +++ b/tests/metagpt/management/test_skill_manager.py @@ -30,7 +30,7 @@ def test_skill_manager(): rsp = manager.retrieve_skill("写测试用例") logger.info(rsp) - assert rsp[0] == 'WriteTest' + assert rsp[0] == "WriteTest" rsp = manager.retrieve_skill_scored("写PRD") logger.info(rsp) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index dc5540520..9682ba760 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -2,11 +2,11 @@ # -*- coding: utf-8 -*- # @Desc : unittest of `metagpt/memory/longterm_memory.py` -from metagpt.config import CONFIG -from metagpt.schema import Message from metagpt.actions import BossRequirement -from metagpt.roles.role import RoleContext +from metagpt.config import CONFIG from metagpt.memory import LongTermMemory +from metagpt.roles.role import RoleContext +from metagpt.schema import Message def test_ltm_search(): @@ -14,25 +14,25 @@ def test_ltm_search(): openai_api_key = CONFIG.openai_api_key assert len(openai_api_key) > 20 - role_id = 'UTUserLtm(Product Manager)' + role_id = "UTUserLtm(Product Manager)" rc = RoleContext(watch=[BossRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) - idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + message = Message(role="BOSS", content=idea, cause_by=BossRequirement) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -47,8 +47,8 @@ def test_ltm_search(): news = ltm_new.find_news([sim_message]) assert len(news) == 0 - new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a Battle City" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 6bb3e8f1d..8b338a79e 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -4,17 +4,16 @@ from typing import List +from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions.action_output import ActionOutput from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message -from metagpt.actions import BossRequirement -from metagpt.actions import WritePRD -from metagpt.actions.action_output import ActionOutput def test_idea_message(): - idea = 'Write a cli snake game' - role_id = 'UTUser1(Product Manager)' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + role_id = "UTUser1(Product Manager)" + message = Message(role="BOSS", content=idea, cause_by=BossRequirement) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -23,13 +22,13 @@ def test_idea_message(): memory_storage.add(message) assert memory_storage.is_initialized is True - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement) new_messages = memory_storage.search(sim_message) - assert len(new_messages) == 0 # similar, return [] + assert len(new_messages) == 0 # similar, return [] - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -38,22 +37,15 @@ def test_idea_message(): def test_actionout_message(): - out_mapping = { - 'field1': (str, ...), - 'field2': (List[str], ...) - } - out_data = { - 'field1': 'field1 value', - 'field2': ['field2 value1', 'field2 value2'] - } - ic_obj = ActionOutput.create_model_class('prd', out_mapping) - - role_id = 'UTUser2(Architect)' - content = 'The boss has requested the creation of a command-line interface (CLI) snake game' - message = Message(content=content, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) # WritePRD as test action + out_mapping = {"field1": (str, ...), "field2": (List[str], ...)} + out_data = {"field1": "field1 value", "field2": ["field2 value1", "field2 value2"]} + ic_obj = ActionOutput.create_model_class("prd", out_mapping) + + role_id = "UTUser2(Architect)" + content = "The boss has requested the creation of a command-line interface (CLI) snake game" + message = Message( + content=content, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD + ) # WritePRD as test action memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -62,19 +54,13 @@ def test_actionout_message(): memory_storage.add(message) assert memory_storage.is_initialized is True - sim_conent = 'The request is command-line interface (CLI) snake game' - sim_message = Message(content=sim_conent, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) + sim_conent = "The request is command-line interface (CLI) snake game" + sim_message = Message(content=sim_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD) new_messages = memory_storage.search(sim_message) - assert len(new_messages) == 0 # similar, return [] + assert len(new_messages) == 0 # similar, return [] - new_conent = 'Incorporate basic features of a snake game such as scoring and increasing difficulty' - new_message = Message(content=new_conent, - instruct_content=ic_obj(**out_data), - role='user', - cause_by=WritePRD) + new_conent = "Incorporate basic features of a snake game such as scoring and increasing difficulty" + new_message = Message(content=new_conent, instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content diff --git a/tests/metagpt/provider/test_base_gpt_api.py b/tests/metagpt/provider/test_base_gpt_api.py index 882338a01..6cfe3b02d 100644 --- a/tests/metagpt/provider/test_base_gpt_api.py +++ b/tests/metagpt/provider/test_base_gpt_api.py @@ -10,6 +10,6 @@ def test_message(): - message = Message(role='user', content='wtf') - assert 'role' in message.to_dict() - assert 'user' in str(message) + message = Message(role="user", content="wtf") + assert "role" in message.to_dict() + assert "user" in str(message) diff --git a/tests/metagpt/provider/test_spark_api.py b/tests/metagpt/provider/test_spark_api.py index bfa2bf76f..3b3dd67f4 100644 --- a/tests/metagpt/provider/test_spark_api.py +++ b/tests/metagpt/provider/test_spark_api.py @@ -6,6 +6,6 @@ def test_message(): llm = SparkAPI() logger.info(llm.ask('只回答"收到了"这三个字。')) - result = llm.ask('写一篇五百字的日记') + result = llm.ask("写一篇五百字的日记") logger.info(result) assert len(result) > 100 diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 52fc4a3c1..1b02fbaa5 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -71,7 +71,7 @@ ``` ''' -SYSTEM_DESIGN = '''## Python package name +SYSTEM_DESIGN = """## Python package name ```python "smart_search_engine" ``` @@ -149,10 +149,10 @@ class KnowledgeBase { S-->>SE: return summary SE-->>M: return summary ``` -''' +""" -TASKS = '''## Logic Analysis +TASKS = """## Logic Analysis 在这个项目中，所有的模块都依赖于“SearchEngine”类，这是主入口，其他的模块（Index、Ranking和Summary）都通过它交互。另外，"Index"类又依赖于"KnowledgeBase"类，因为它需要从知识库中获取数据。 @@ -181,7 +181,7 @@ class KnowledgeBase { ] ``` 这个任务列表首先定义了最基础的模块，然后是依赖这些模块的模块，最后是辅助模块。可以根据团队的能力和资源，同时开发多个任务，只要满足依赖关系。例如，在开发"search.py"之前，可以同时开发"knowledge_base.py"、"index.py"、"ranking.py"和"summary.py"。 -''' +""" TASKS_TOMATO_CLOCK = '''## Required Python third-party packages: Provided in requirements.txt format @@ -224,30 +224,30 @@ class KnowledgeBase { TASK = """smart_search_engine/knowledge_base.py""" STRS_FOR_PARSING = [ -""" + """ ## 1 ```python a ``` """, -""" + """ ##2 ```python "a" ``` """, -""" + """ ## 3 ```python a = "a" ``` """, -""" + """ ## 4 ```python a = 'a' ``` -""" +""", ] diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index c0c48d0b1..f44188c17 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -35,13 +35,13 @@ def test_parse_str(): for idx, i in enumerate(STRS_FOR_PARSING): text = CodeParser.parse_str(f"{idx+1}", i) # logger.info(text) - assert text == 'a' + assert text == "a" def test_parse_blocks(): tasks = CodeParser.parse_blocks(TASKS) logger.info(tasks.keys()) - assert 'Task list' in tasks.keys() + assert "Task list" in tasks.keys() target_list = [ diff --git a/tests/metagpt/roles/test_invoice_ocr_assistant.py b/tests/metagpt/roles/test_invoice_ocr_assistant.py index 75097e73c..c9aad93a7 100644 --- a/tests/metagpt/roles/test_invoice_ocr_assistant.py +++ b/tests/metagpt/roles/test_invoice_ocr_assistant.py @@ -9,8 +9,8 @@ from pathlib import Path -import pytest import pandas as pd +import pytest from metagpt.roles.invoice_ocr_assistant import InvoiceOCRAssistant from metagpt.schema import Message @@ -24,82 +24,39 @@ "Invoicing date", Path("../../data/invoices/invoice-1.pdf"), Path("../../../data/invoice_table/invoice-1.xlsx"), - [ - { - "收款人": "小明", - "城市": "深圳市", - "总费用/元": 412.00, - "开票日期": "2023年02月03日" - } - ] + [{"收款人": "小明", "城市": "深圳市", "总费用/元": 412.00, "开票日期": "2023年02月03日"}], ), ( "Invoicing date", Path("../../data/invoices/invoice-2.png"), Path("../../../data/invoice_table/invoice-2.xlsx"), - [ - { - "收款人": "铁头", - "城市": "广州市", - "总费用/元": 898.00, - "开票日期": "2023年03月17日" - } - ] + [{"收款人": "铁头", "城市": "广州市", "总费用/元": 898.00, "开票日期": "2023年03月17日"}], ), ( "Invoicing date", Path("../../data/invoices/invoice-3.jpg"), Path("../../../data/invoice_table/invoice-3.xlsx"), - [ - { - "收款人": "夏天", - "城市": "福州市", - "总费用/元": 2462.00, - "开票日期": "2023年08月26日" - } - ] + [{"收款人": "夏天", "城市": "福州市", "总费用/元": 2462.00, "开票日期": "2023年08月26日"}], ), ( "Invoicing date", Path("../../data/invoices/invoice-4.zip"), Path("../../../data/invoice_table/invoice-4.xlsx"), [ - { - "收款人": "小明", - "城市": "深圳市", - "总费用/元": 412.00, - "开票日期": "2023年02月03日" - }, - { - "收款人": "铁头", - "城市": "广州市", - "总费用/元": 898.00, - "开票日期": "2023年03月17日" - }, - { - "收款人": "夏天", - "城市": "福州市", - "总费用/元": 2462.00, - "开票日期": "2023年08月26日" - } - ] + {"收款人": "小明", "城市": "深圳市", "总费用/元": 412.00, "开票日期": "2023年02月03日"}, + {"收款人": "铁头", "城市": "广州市", "总费用/元": 898.00, "开票日期": "2023年03月17日"}, + {"收款人": "夏天", "城市": "福州市", "总费用/元": 2462.00, "开票日期": "2023年08月26日"}, + ], ), - ] + ], ) async def test_invoice_ocr_assistant( - query: str, - invoice_path: Path, - invoice_table_path: Path, - expected_result: list[dict] + query: str, invoice_path: Path, invoice_table_path: Path, expected_result: list[dict] ): invoice_path = Path.cwd() / invoice_path role = InvoiceOCRAssistant() - await role.run(Message( - content=query, - instruct_content={"file_path": invoice_path} - )) + await role.run(Message(content=query, instruct_content={"file_path": invoice_path})) invoice_table_path = Path.cwd() / invoice_table_path df = pd.read_excel(invoice_table_path) - dict_result = df.to_dict(orient='records') + dict_result = df.to_dict(orient="records") assert dict_result == expected_result - diff --git a/tests/metagpt/roles/test_researcher.py b/tests/metagpt/roles/test_researcher.py index 01b5dae3b..dd130662d 100644 --- a/tests/metagpt/roles/test_researcher.py +++ b/tests/metagpt/roles/test_researcher.py @@ -11,10 +11,12 @@ async def mock_llm_ask(self, prompt: str, system_msgs): if "Please provide up to 2 necessary keywords" in prompt: return '["dataiku", "datarobot"]' elif "Provide up to 4 queries related to your research topic" in prompt: - return '["Dataiku machine learning platform", "DataRobot AI platform comparison", ' \ + return ( + '["Dataiku machine learning platform", "DataRobot AI platform comparison", ' '"Dataiku vs DataRobot features", "Dataiku and DataRobot use cases"]' + ) elif "sort the remaining search results" in prompt: - return '[1,2]' + return "[1,2]" elif "Not relevant." in prompt: return "Not relevant" if random() > 0.5 else prompt[-100:] elif "provide a detailed research report" in prompt: diff --git a/tests/metagpt/roles/test_tutorial_assistant.py b/tests/metagpt/roles/test_tutorial_assistant.py index 945620cfc..105f976c3 100644 --- a/tests/metagpt/roles/test_tutorial_assistant.py +++ b/tests/metagpt/roles/test_tutorial_assistant.py @@ -12,10 +12,7 @@ @pytest.mark.asyncio -@pytest.mark.parametrize( - ("language", "topic"), - [("Chinese", "Write a tutorial about Python")] -) +@pytest.mark.parametrize(("language", "topic"), [("Chinese", "Write a tutorial about Python")]) async def test_tutorial_assistant(language: str, topic: str): topic = "Write a tutorial about MySQL" role = TutorialAssistant(language=language) @@ -24,4 +21,4 @@ async def test_tutorial_assistant(language: str, topic: str): title = filename.split("/")[-1].split(".")[0] async with aiofiles.open(filename, mode="r") as reader: content = await reader.read() - assert content.startswith(f"# {title}") \ No newline at end of file + assert content.startswith(f"# {title}") diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index 285bff323..2d9cb85c9 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -2,9 +2,8 @@ # @Date : 2023/7/22 02:40 # @Author : stellahong (stellahong@fuzhi.ai) # -from metagpt.software_company import SoftwareCompany from metagpt.roles import ProductManager - +from metagpt.software_company import SoftwareCompany from tests.metagpt.roles.ui_role import UI diff --git a/tests/metagpt/test_gpt.py b/tests/metagpt/test_gpt.py index 89dd726a8..285e8134c 100644 --- a/tests/metagpt/test_gpt.py +++ b/tests/metagpt/test_gpt.py @@ -14,7 +14,7 @@ @pytest.mark.usefixtures("llm_api") class TestGPT: def test_llm_api_ask(self, llm_api): - answer = llm_api.ask('hello chatgpt') + answer = llm_api.ask("hello chatgpt") assert len(answer) > 0 # def test_gptapi_ask_batch(self, llm_api): @@ -22,22 +22,22 @@ def test_llm_api_ask(self, llm_api): # assert len(answer) > 0 def test_llm_api_ask_code(self, llm_api): - answer = llm_api.ask_code(['请扮演一个Google Python专家工程师，如果理解，回复明白', '写一个hello world']) + answer = llm_api.ask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask(self, llm_api): - answer = await llm_api.aask('hello chatgpt') + answer = await llm_api.aask("hello chatgpt") assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask_code(self, llm_api): - answer = await llm_api.aask_code(['请扮演一个Google Python专家工程师，如果理解，回复明白', '写一个hello world']) + answer = await llm_api.aask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_costs(self, llm_api): - await llm_api.aask('hello chatgpt') + await llm_api.aask("hello chatgpt") costs = llm_api.get_costs() logger.info(costs) assert costs.total_cost > 0 diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 11503af1d..03341212b 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -18,17 +18,17 @@ def llm(): @pytest.mark.asyncio async def test_llm_aask(llm): - assert len(await llm.aask('hello world')) > 0 + assert len(await llm.aask("hello world")) > 0 @pytest.mark.asyncio async def test_llm_aask_batch(llm): - assert len(await llm.aask_batch(['hi', 'write python hello world.'])) > 0 + assert len(await llm.aask_batch(["hi", "write python hello world."])) > 0 @pytest.mark.asyncio async def test_llm_acompletion(llm): - hello_msg = [{'role': 'user', 'content': 'hello'}] + hello_msg = [{"role": "user", "content": "hello"}] assert len(await llm.acompletion(hello_msg)) > 0 assert len(await llm.acompletion_batch([hello_msg])) > 0 assert len(await llm.acompletion_batch_text([hello_msg])) > 0 diff --git a/tests/metagpt/test_message.py b/tests/metagpt/test_message.py index e26f38381..ae6708943 100644 --- a/tests/metagpt/test_message.py +++ b/tests/metagpt/test_message.py @@ -11,26 +11,26 @@ def test_message(): - msg = Message(role='User', content='WTF') - assert msg.to_dict()['role'] == 'User' - assert 'User' in str(msg) + msg = Message(role="User", content="WTF") + assert msg.to_dict()["role"] == "User" + assert "User" in str(msg) def test_all_messages(): - test_content = 'test_message' + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] for msg in msgs: assert msg.content == test_content def test_raw_message(): - msg = RawMessage(role='user', content='raw') - assert msg['role'] == 'user' - assert msg['content'] == 'raw' + msg = RawMessage(role="user", content="raw") + assert msg["role"] == "user" + assert msg["content"] == "raw" with pytest.raises(KeyError): - assert msg['1'] == 1, "KeyError: '1'" + assert msg["1"] == 1, "KeyError: '1'" diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 11fd804ec..22cfa58a4 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -9,6 +9,6 @@ def test_role_desc(): - i = Role(profile='Sales', desc='Best Seller') - assert i.profile == 'Sales' - assert i._setting.desc == 'Best Seller' + i = Role(profile="Sales", desc="Best Seller") + assert i.profile == "Sales" + assert i._setting.desc == "Best Seller" diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 12666e0d3..c154d77e1 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -9,13 +9,13 @@ def test_messages(): - test_content = 'test_message' + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] text = str(msgs) - roles = ['user', 'system', 'assistant', 'QA'] + roles = ["user", "system", "assistant", "QA"] assert all([i in text for i in roles]) diff --git a/tests/metagpt/tools/test_code_interpreter.py b/tests/metagpt/tools/test_code_interpreter.py index 0eec3f80b..03d4ce8df 100644 --- a/tests/metagpt/tools/test_code_interpreter.py +++ b/tests/metagpt/tools/test_code_interpreter.py @@ -1,23 +1,22 @@ -import pytest -import pandas as pd from pathlib import Path -from tests.data import sales_desc, store_desc -from metagpt.tools.code_interpreter import OpenCodeInterpreter, OpenInterpreterDecorator +import pandas as pd +import pytest + from metagpt.actions import Action from metagpt.logs import logger +from metagpt.tools.code_interpreter import OpenCodeInterpreter, OpenInterpreterDecorator - -logger.add('./tests/data/test_ci.log') +logger.add("./tests/data/test_ci.log") stock = "./tests/data/baba_stock.csv" # TODO: 需要一种表格数据格式，能够支持schame管理的，标注字段类型和字段含义。 class CreateStockIndicators(Action): @OpenInterpreterDecorator(save_code=True, code_file_path="./tests/data/stock_indicators.py") - async def run(self, stock_path: str, indicators=['Simple Moving Average', 'BollingerBands']) -> pd.DataFrame: + async def run(self, stock_path: str, indicators=["Simple Moving Average", "BollingerBands"]) -> pd.DataFrame: """对stock_path中的股票数据, 使用pandas和ta计算indicators中的技术指标, 返回带有技术指标的股票数据，不需要去除空值, 不需要安装任何包； - 指标生成对应的三列: SMA, BB_upper, BB_lower + 指标生成对应的三列: SMA, BB_upper, BB_lower """ ... @@ -25,18 +24,20 @@ async def run(self, stock_path: str, indicators=['Simple Moving Average', 'Bolli @pytest.mark.asyncio async def test_actions(): # 计算指标 - indicators = ['Simple Moving Average', 'BollingerBands'] + indicators = ["Simple Moving Average", "BollingerBands"] stocker = CreateStockIndicators() df, msg = await stocker.run(stock, indicators=indicators) assert isinstance(df, pd.DataFrame) - assert 'Close' in df.columns - assert 'Date' in df.columns + assert "Close" in df.columns + assert "Date" in df.columns # 将df保存为文件，将文件路径传入到下一个action - df_path = './tests/data/stock_indicators.csv' + df_path = "./tests/data/stock_indicators.csv" df.to_csv(df_path) assert Path(df_path).is_file() # 可视化指标结果 - figure_path = './tests/data/figure_ci.png' + figure_path = "./tests/data/figure_ci.png" ci_ploter = OpenCodeInterpreter() - ci_ploter.chat(f"使用seaborn对{df_path}中与股票布林带有关的数据列的Date, Close, SMA, BB_upper（布林带上界）, BB_lower（布林带下界）进行可视化, 可视化图片保存在{figure_path}中。不需要任何指标计算，把Date列转换为日期类型。要求图片优美，BB_upper, BB_lower之间使用合适的颜色填充。") + ci_ploter.chat( + f"使用seaborn对{df_path}中与股票布林带有关的数据列的Date, Close, SMA, BB_upper（布林带上界）, BB_lower（布林带下界）进行可视化, 可视化图片保存在{figure_path}中。不需要任何指标计算，把Date列转换为日期类型。要求图片优美，BB_upper, BB_lower之间使用合适的颜色填充。" + ) assert Path(figure_path).is_file() diff --git a/tests/metagpt/tools/test_prompt_generator.py b/tests/metagpt/tools/test_prompt_generator.py index d2e870c6d..ddbd2c43b 100644 --- a/tests/metagpt/tools/test_prompt_generator.py +++ b/tests/metagpt/tools/test_prompt_generator.py @@ -20,8 +20,9 @@ @pytest.mark.usefixtures("llm_api") def test_gpt_prompt_generator(llm_api): generator = GPTPromptGenerator() - example = "商品名称:WonderLab 新肌果味代餐奶昔 小胖瓶 胶原蛋白升级版 饱腹代餐粉6瓶 75g/瓶(6瓶/盒) 店铺名称:金力宁食品专营店 " \ - "品牌:WonderLab 保质期:1年 产地:中国 净含量:450g" + example = ( + "商品名称:WonderLab 新肌果味代餐奶昔 小胖瓶 胶原蛋白升级版 饱腹代餐粉6瓶 75g/瓶(6瓶/盒) 店铺名称:金力宁食品专营店 " "品牌:WonderLab 保质期:1年 产地:中国 净含量:450g" + ) results = llm_api.ask_batch(generator.gen(example)) logger.info(results) @@ -46,7 +47,7 @@ def test_enron_template(llm_api): results = template.gen(subj) assert len(results) > 0 - assert any("Write an email with the subject \"Meeting Agenda\"." in r for r in results) + assert any('Write an email with the subject "Meeting Agenda".' in r for r in results) def test_beagec_template(): @@ -54,5 +55,6 @@ def test_beagec_template(): results = template.gen() assert len(results) > 0 - assert any("Edit and revise this document to improve its grammar, vocabulary, spelling, and style." - in r for r in results) + assert any( + "Edit and revise this document to improve its grammar, vocabulary, spelling, and style." in r for r in results + ) diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..4edd8fb3b 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,7 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.tools.sd_engine import WORKSPACE_ROOT, SDEngine def test_sd_engine_init(): diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index a7fe063a6..25bce124a 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -16,7 +16,9 @@ class MockSearchEnine: async def run(self, query: str, max_results: int = 8, as_string: bool = True) -> str | list[dict[str, str]]: - rets = [{"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results)] + rets = [ + {"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results) + ] return "\n".join(rets) if as_string else rets @@ -34,10 +36,14 @@ async def run(self, query: str, max_results: int = 8, as_string: bool = True) -> (SearchEngineType.DUCK_DUCK_GO, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), - ], ) -async def test_search_engine(search_engine_typpe, run_func, max_results, as_string, ): +async def test_search_engine( + search_engine_typpe, + run_func, + max_results, + as_string, +): search_engine = SearchEngine(search_engine_typpe, run_func) rsp = await search_engine.run("metagpt", max_results=max_results, as_string=as_string) logger.info(rsp) diff --git a/tests/metagpt/tools/test_search_engine_meilisearch.py b/tests/metagpt/tools/test_search_engine_meilisearch.py index 8d2bb6494..d5f7d162b 100644 --- a/tests/metagpt/tools/test_search_engine_meilisearch.py +++ b/tests/metagpt/tools/test_search_engine_meilisearch.py @@ -13,7 +13,7 @@ from metagpt.logs import logger from metagpt.tools.search_engine_meilisearch import DataSource, MeilisearchEngine -MASTER_KEY = '116Qavl2qpCYNEJNv5-e0RC9kncev1nr1gt7ybEGVLk' +MASTER_KEY = "116Qavl2qpCYNEJNv5-e0RC9kncev1nr1gt7ybEGVLk" @pytest.fixture() @@ -29,7 +29,7 @@ def test_meilisearch(search_engine_server): search_engine = MeilisearchEngine(url="http://localhost:7700", token=MASTER_KEY) # 假设有一个名为"books"的数据源，包含要添加的文档库 - books_data_source = DataSource(name='books', url='https://example.com/books') + books_data_source = DataSource(name="books", url="https://example.com/books") # 假设有一个名为"documents"的文档库，包含要添加的文档 documents = [ @@ -43,4 +43,4 @@ def test_meilisearch(search_engine_server): # 添加文档库到搜索引擎 search_engine.add_documents(books_data_source, documents) - logger.info(search_engine.search('Book 1')) + logger.info(search_engine.search("Book 1")) diff --git a/tests/metagpt/tools/test_summarize.py b/tests/metagpt/tools/test_summarize.py index cf616c144..6a372defb 100644 --- a/tests/metagpt/tools/test_summarize.py +++ b/tests/metagpt/tools/test_summarize.py @@ -20,7 +20,6 @@ 1. 请根据上下文，对用户搜索请求进行总结性回答，不要包括与请求无关的文本 2. 以 [正文](引用链接) markdown形式在正文中**自然标注**~5个文本（如商品词或类似文本段），以便跳转 3. 回复优雅、清晰，**绝不重复文本**，行文流畅，长度居中""", - """# 上下文 [{'title': '去厦门 有哪些推荐的美食？ - 知乎', 'href': 'https://www.zhihu.com/question/286901854', 'body': '知乎，中文互联网高质量的问答社区和创作者聚集的原创内容平台，于 2011 年 1 月正式上线，以「让人们更好的分享知识、经验和见解，找到自己的解答」为品牌使命。知乎凭借认真、专业、友善的社区氛围、独特的产品机制以及结构化和易获得的优质内容，聚集了中文互联网科技、商业、影视 ...'}, {'title': '厦门到底有哪些真正值得吃的美食？ - 知乎', 'href': 'https://www.zhihu.com/question/38012322', 'body': '有几个特色菜在别处不太能吃到，值得一试~常点的有西多士、沙茶肉串、咕老肉（个人认为还是良山排档的更炉火纯青~），因为爱吃芋泥，每次还会点一个芋泥鸭~人均50元左右. 潮福城. 厦门这两年经营港式茶点的店越来越多，但是最经典的还是潮福城的茶点 ...'}, {'title': '超全厦门美食攻略，好吃不贵不踩雷 - 知乎 - 知乎专栏', 'href': 'https://zhuanlan.zhihu.com/p/347055615', 'body': '厦门老字号店铺，味道卫生都有保障，喜欢吃芒果的，不要错过芒果牛奶绵绵冰. 285蚝味馆 70/人. 上过《舌尖上的中国》味道不用多说，想吃地道的海鲜烧烤就来这里. 堂宴.老厦门私房菜 80/人. 非常多的明星打卡过，上过《十二道锋味》，吃厦门传统菜的好去处 ...'}, {'title': '福建名小吃||寻味厦门，十大特色名小吃，你都吃过哪几样？ - 知乎', 'href': 'https://zhuanlan.zhihu.com/p/375781836', 'body': '第一期，分享厦门的特色美食。 厦门是一个风景旅游城市，许多人来到厦门，除了游览厦门独特的风景之外，最难忘的应该是厦门的特色小吃。厦门小吃多种多样，有到厦门必吃的沙茶面、米线糊、蚵仔煎、土笋冻等非常之多。那么，厦门的名小吃有哪些呢？'}, {'title': '大家如果去厦门旅游的话，好吃的有很多，但... 来自庄时利和 - 微博', 'href': 'https://weibo.com/1728715190/MEAwzscRT', 'body': '大家如果去厦门旅游的话，好吃的有很多，但如果只选一样的话，我个人会选择莲花煎蟹。 靠海吃海，吃蟹对于闽南人来说是很平常的一件事。 厦门传统的做法多是清蒸或水煮，上世纪八十年代有一同安人在厦门的莲花公园旁，摆摊做起了煎蟹的生意。'}, {'title': '厦门美食,厦门美食攻略,厦门旅游美食攻略 - 马蜂窝', 'href': 'https://www.mafengwo.cn/cy/10132/gonglve.html', 'body': '醉壹号海鲜大排档 (厦门美食地标店) No.3. 哆啦Eanny 的最新点评：. 环境 挺复古的闽南风情，花砖地板，一楼有海鲜自己点菜，二楼室内位置，三楼露天位置，环境挺不错的。. 苦螺汤，看起来挺清的，螺肉吃起来很脆。. 姜... 5.0 分. 482 条用户点评.'}, {'title': '厦门超强中山路小吃合集，29家本地人推荐的正宗美食 - 马蜂窝', 'href': 'https://www.mafengwo.cn/gonglve/ziyouxing/176485.html', 'body': '莲欢海蛎煎. 提到厦门就想到海蛎煎，而这家位于中山路局口街的莲欢海蛎煎是实打实的好吃!. ·局口街老巷之中，全室外环境，吃的就是这种感觉。. ·取名"莲欢"，是希望妻子每天开心。. 新鲜的食材，实在的用料，这样的用心也定能讨食客欢心。. ·海蛎又 ...'}, {'title': '厦门市 10 大餐厅- Tripadvisor', 'href': 'https://cn.tripadvisor.com/Restaurants-g297407-Xiamen_Fujian.html', 'body': '厦门市餐厅：在Tripadvisor查看中国厦门市餐厅的点评，并以价格、地点及更多选项进行搜索。 ... "牛排太好吃了啊啊啊" ... "厦门地区最老品牌最有口碑的潮州菜餐厅" ...'}, {'title': '#福建10条美食街简直不要太好吃#每到一... 来自新浪厦门 - 微博', 'href': 'https://weibo.com/1740522895/MF1lY7W4n', 'body': '福建的这10条美食街，你一定不能错过!福州师大学生街、福州达明路美食街、厦门八市、漳州古城老街、宁德老南门电影院美食集市、龙岩中山路美食街、三明龙岗夜市、莆田金鼎夜市、莆田玉湖夜市、南平嘉禾美食街。世间万事皆难，唯有美食可以治愈一切。'}, {'title': '厦门这50家餐厅最值得吃 - 腾讯新闻', 'href': 'https://new.qq.com/rain/a/20200114A09HJT00', 'body': '没有什么事是一顿辣解决不了的! 创意辣、川湘辣、温柔辣、异域辣，芙蓉涧的菜能把辣椒玩出花来! ... 早在2005年，这家老牌的东南亚餐厅就开在厦门莲花了，在许多老厦门的心中，都觉得这里有全厦门最好吃的咖喱呢。 ...'}, {'title': '好听的美食？又好听又好吃的食物有什么？ - 哔哩哔哩', 'href': 'https://www.bilibili.com/read/cv23430069/', 'body': '专栏 / 好听的美食？又好听又好吃的食物有什么？ 又好听又好吃的食物有什么？ 2023-05-02 18:01 --阅读 · --喜欢 · --评论'}] @@ -31,7 +30,7 @@ 你是专业管家团队的一员，会给出有帮助的建议 1. 请根据上下文，对用户搜索请求进行总结性回答，不要包括与请求无关的文本 2. 以 [正文](引用链接) markdown形式在正文中**自然标注**3-5个文本（如商品词或类似文本段），以便跳转 -3. 回复优雅、清晰，**绝不重复文本**，行文流畅，长度居中""" +3. 回复优雅、清晰，**绝不重复文本**，行文流畅，长度居中""", ] diff --git a/tests/metagpt/tools/test_translate.py b/tests/metagpt/tools/test_translate.py index 47a9034a5..024bda3ca 100644 --- a/tests/metagpt/tools/test_translate.py +++ b/tests/metagpt/tools/test_translate.py @@ -16,7 +16,7 @@ def test_translate(llm_api): poetries = [ ("Let life be beautiful like summer flowers", "花"), - ("The ancient Chinese poetries are all songs.", "中国") + ("The ancient Chinese poetries are all songs.", "中国"), ] for i, j in poetries: prompt = Translator.translate_prompt(i) diff --git a/tests/metagpt/tools/test_ut_generator.py b/tests/metagpt/tools/test_ut_generator.py index 6f29999d4..2ae94885f 100644 --- a/tests/metagpt/tools/test_ut_generator.py +++ b/tests/metagpt/tools/test_ut_generator.py @@ -16,8 +16,12 @@ def test_api_to_ut_sample(self): tags = ["测试"] # "智能合同导入", "律师审查", "ai合同审查", "草拟合同&律师在线审查", "合同审批", "履约管理", "签约公司"] # 这里在文件中手动加入了两个测试标签的API - utg = UTGenerator(swagger_file=swagger_file, ut_py_path=UT_PY_PATH, questions_path=API_QUESTIONS_PATH, - template_prefix=YFT_PROMPT_PREFIX) + utg = UTGenerator( + swagger_file=swagger_file, + ut_py_path=UT_PY_PATH, + questions_path=API_QUESTIONS_PATH, + template_prefix=YFT_PROMPT_PREFIX, + ) ret = utg.generate_ut(include_tags=tags) # 后续加入对文件生成内容与数量的检验 assert ret diff --git a/tests/metagpt/utils/test_code_parser.py b/tests/metagpt/utils/test_code_parser.py index 707b558e1..6b7349cd9 100644 --- a/tests/metagpt/utils/test_code_parser.py +++ b/tests/metagpt/utils/test_code_parser.py @@ -131,10 +131,10 @@ def test_parse_str(self, parser, text): def test_parse_file_list(self, parser, text): result = parser.parse_file_list("Task list", text) print(result) - assert result == ['task1', 'task2'] + assert result == ["task1", "task2"] -if __name__ == '__main__': +if __name__ == "__main__": t = TestCodeParser() t.test_parse_file_list(CodeParser(), t_text) # TestCodeParser.test_parse_file_list() diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index ec4443175..d3837ca8f 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -16,12 +16,12 @@ class TestGetProjectRoot: def change_etc_dir(self): # current_directory = Path.cwd() - abs_root = '/etc' + abs_root = "/etc" os.chdir(abs_root) def test_get_project_root(self): project_root = get_project_root() - assert project_root.name == 'metagpt' + assert project_root.name == "metagpt" def test_get_root_exception(self): with pytest.raises(Exception) as exc_info: diff --git a/tests/metagpt/utils/test_config.py b/tests/metagpt/utils/test_config.py index 558a4e5a4..b68a535f9 100644 --- a/tests/metagpt/utils/test_config.py +++ b/tests/metagpt/utils/test_config.py @@ -20,12 +20,12 @@ def test_config_class_is_singleton(): def test_config_class_get_key_exception(): with pytest.raises(Exception) as exc_info: config = Config() - config.get('wtf') + config.get("wtf") assert str(exc_info.value) == "Key 'wtf' not found in environment variables or in the YAML file" def test_config_yaml_file_not_exists(): - config = Config('wtf.yaml') + config = Config("wtf.yaml") with pytest.raises(Exception) as exc_info: - config.get('OPENAI_BASE_URL') + config.get("OPENAI_BASE_URL") assert str(exc_info.value) == "Key 'OPENAI_BASE_URL' not found in environment variables or in the YAML file" diff --git a/tests/metagpt/utils/test_custom_aio_session.py b/tests/metagpt/utils/test_custom_aio_session.py index 3a8a7bf7e..e2876e4b8 100644 --- a/tests/metagpt/utils/test_custom_aio_session.py +++ b/tests/metagpt/utils/test_custom_aio_session.py @@ -10,12 +10,12 @@ async def try_hello(api): - batch = [[{'role': 'user', 'content': 'hello'}]] + batch = [[{"role": "user", "content": "hello"}]] results = await api.acompletion_batch_text(batch) return results async def aask_batch(api: OpenAIGPTAPI): - results = await api.aask_batch(['hi', 'write python hello world.']) + results = await api.aask_batch(["hi", "write python hello world."]) logger.info(results) return results diff --git a/tests/metagpt/utils/test_file.py b/tests/metagpt/utils/test_file.py index b30e6be93..83e317213 100644 --- a/tests/metagpt/utils/test_file.py +++ b/tests/metagpt/utils/test_file.py @@ -15,12 +15,11 @@ @pytest.mark.asyncio @pytest.mark.parametrize( ("root_path", "filename", "content"), - [(Path("/code/MetaGPT/data/tutorial_docx/2023-09-07_17-05-20"), "test.md", "Hello World!")] + [(Path("/code/MetaGPT/data/tutorial_docx/2023-09-07_17-05-20"), "test.md", "Hello World!")], ) async def test_write_and_read_file(root_path: Path, filename: str, content: bytes): - full_file_name = await File.write(root_path=root_path, filename=filename, content=content.encode('utf-8')) + full_file_name = await File.write(root_path=root_path, filename=filename, content=content.encode("utf-8")) assert isinstance(full_file_name, Path) assert root_path / filename == full_file_name file_data = await File.read(full_file_name) assert file_data.decode("utf-8") == content - diff --git a/tests/metagpt/utils/test_output_parser.py b/tests/metagpt/utils/test_output_parser.py index 4e362f9f7..7a3aedbe8 100644 --- a/tests/metagpt/utils/test_output_parser.py +++ b/tests/metagpt/utils/test_output_parser.py @@ -14,17 +14,17 @@ def test_parse_blocks(): test_text = "##block1\nThis is block 1.\n##block2\nThis is block 2." - expected_result = {'block1': 'This is block 1.', 'block2': 'This is block 2.'} + expected_result = {"block1": "This is block 1.", "block2": "This is block 2."} assert OutputParser.parse_blocks(test_text) == expected_result def test_parse_code(): test_text = "```python\nprint('Hello, world!')```" expected_result = "print('Hello, world!')" - assert OutputParser.parse_code(test_text, 'python') == expected_result + assert OutputParser.parse_code(test_text, "python") == expected_result with pytest.raises(Exception): - OutputParser.parse_code(test_text, 'java') + OutputParser.parse_code(test_text, "java") def test_parse_python_code(): @@ -45,13 +45,13 @@ def test_parse_python_code(): def test_parse_str(): test_text = "name = 'Alice'" - expected_result = 'Alice' + expected_result = "Alice" assert OutputParser.parse_str(test_text) == expected_result def test_parse_file_list(): test_text = "files=['file1', 'file2', 'file3']" - expected_result = ['file1', 'file2', 'file3'] + expected_result = ["file1", "file2", "file3"] assert OutputParser.parse_file_list(test_text) == expected_result with pytest.raises(Exception): @@ -60,7 +60,7 @@ def test_parse_file_list(): def test_parse_data(): test_data = "##block1\n```python\nprint('Hello, world!')\n```\n##block2\nfiles=['file1', 'file2', 'file3']" - expected_result = {'block1': "print('Hello, world!')", 'block2': ['file1', 'file2', 'file3']} + expected_result = {"block1": "print('Hello, world!')", "block2": ["file1", "file2", "file3"]} assert OutputParser.parse_data(test_data) == expected_result @@ -103,9 +103,11 @@ def test_parse_data(): None, Exception, ), - ] + ], ) -def test_extract_struct(text: str, data_type: Union[type(list), type(dict)], parsed_data: Union[list, dict], expected_exception): +def test_extract_struct( + text: str, data_type: Union[type(list), type(dict)], parsed_data: Union[list, dict], expected_exception +): def case(): resp = OutputParser.extract_struct(text, data_type) assert resp == parsed_data @@ -117,7 +119,7 @@ def case(): case() -if __name__ == '__main__': +if __name__ == "__main__": t_text = ''' ## Required Python third-party packages ```python @@ -216,7 +218,7 @@ def case(): "Requirement Pool": (List[Tuple[str, str]], ...), "Anything UNCLEAR": (str, ...), } - t_text1 = '''## Original Requirements: + t_text1 = """## Original Requirements: The boss wants to create a web-based version of the game "Fly Bird". @@ -284,7 +286,7 @@ def case(): ## Anything UNCLEAR: There are no unclear points. - ''' + """ d = OutputParser.parse_data_with_mapping(t_text1, OUTPUT_MAPPING) import json diff --git a/tests/metagpt/utils/test_parse_html.py b/tests/metagpt/utils/test_parse_html.py index 42be416a6..dd15bd80b 100644 --- a/tests/metagpt/utils/test_parse_html.py +++ b/tests/metagpt/utils/test_parse_html.py @@ -52,9 +52,11 @@ """ -CONTENT = 'This is a HeadingThis is a paragraph witha linkand someemphasizedtext.Item 1Item 2Item 3Numbered Item 1Numbered '\ -'Item 2Numbered Item 3Header 1Header 2Row 1, Cell 1Row 1, Cell 2Row 2, Cell 1Row 2, Cell 2Name:Email:SubmitThis is a div '\ -'with a class "box".a link' +CONTENT = ( + "This is a HeadingThis is a paragraph witha linkand someemphasizedtext.Item 1Item 2Item 3Numbered Item 1Numbered " + "Item 2Numbered Item 3Header 1Header 2Row 1, Cell 1Row 1, Cell 2Row 2, Cell 1Row 2, Cell 2Name:Email:SubmitThis is a div " + 'with a class "box".a link' +) def test_web_page(): diff --git a/tests/metagpt/utils/test_pycst.py b/tests/metagpt/utils/test_pycst.py index 07352eac2..9cf876611 100644 --- a/tests/metagpt/utils/test_pycst.py +++ b/tests/metagpt/utils/test_pycst.py @@ -1,6 +1,6 @@ from metagpt.utils import pycst -code = ''' +code = """ #!/usr/bin/env python # -*- coding: utf-8 -*- from typing import overload @@ -24,7 +24,7 @@ def __init__(self, name: str, age: int): def greet(self): return f"Hello, my name is {self.name} and I am {self.age} years old." -''' +""" documented_code = ''' """ diff --git a/tests/metagpt/utils/test_text.py b/tests/metagpt/utils/test_text.py index 0caf8abaa..7003c7767 100644 --- a/tests/metagpt/utils/test_text.py +++ b/tests/metagpt/utils/test_text.py @@ -29,7 +29,7 @@ def _paragraphs(n): (_msgs(), "gpt-4", "Hello," * 1000, 2000, 2), (_msgs(), "gpt-4-32k", "System", 4000, 14), (_msgs(), "gpt-4-32k", "Hello," * 2000, 4000, 12), - ] + ], ) def test_reduce_message_length(msgs, model_name, system_text, reserved, expected): assert len(reduce_message_length(msgs, model_name, system_text, reserved)) / (len("Hello,")) / 1000 == expected @@ -42,7 +42,7 @@ def test_reduce_message_length(msgs, model_name, system_text, reserved, expected (" ".join("Hello World." for _ in range(1000)), "Prompt: {}", "gpt-3.5-turbo-16k", "System", 3000, 1), (" ".join("Hello World." for _ in range(4000)), "Prompt: {}", "gpt-4", "System", 2000, 2), (" ".join("Hello World." for _ in range(8000)), "Prompt: {}", "gpt-4-32k", "System", 4000, 1), - ] + ], ) def test_generate_prompt_chunk(text, prompt_template, model_name, system_text, reserved, expected): ret = list(generate_prompt_chunk(text, prompt_template, model_name, system_text, reserved)) @@ -58,7 +58,7 @@ def test_generate_prompt_chunk(text, prompt_template, model_name, system_text, r ("......", ".", 2, ["...", "..."]), ("......", ".", 3, ["..", "..", ".."]), (".......", ".", 2, ["....", "..."]), - ] + ], ) def test_split_paragraph(paragraph, sep, count, expected): ret = split_paragraph(paragraph, sep, count) @@ -71,7 +71,7 @@ def test_split_paragraph(paragraph, sep, count, expected): ("Hello\\nWorld", "Hello\nWorld"), ("Hello\\tWorld", "Hello\tWorld"), ("Hello\\u0020World", "Hello World"), - ] + ], ) def test_decode_unicode_escape(text, expected): assert decode_unicode_escape(text) == expected From 2bf8ef8c6ad18808447b827b6699e89650d7170c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 17:08:00 +0800 Subject: [PATCH 123/232] feat: RFC 135 --- metagpt/actions/design_api.py | 47 +++++++- metagpt/actions/prepare_documents.py | 38 +++++-- metagpt/actions/write_prd.py | 38 ++++++- metagpt/config.py | 3 +- metagpt/const.py | 5 + metagpt/environment.py | 9 +- metagpt/roles/product_manager.py | 7 +- metagpt/schema.py | 40 ++++++- metagpt/utils/dependency_file.py | 83 ++++++++++++++ metagpt/utils/file_repository.py | 116 ++++++++++++-------- metagpt/utils/git_repository.py | 15 ++- requirements.txt | 2 +- startup.py | 6 + tests/metagpt/utils/test_dependency_file.py | 64 +++++++++++ tests/metagpt/utils/test_file_repository.py | 10 +- tests/metagpt/utils/test_git_repository.py | 15 +++ 16 files changed, 416 insertions(+), 82 deletions(-) create mode 100644 metagpt/utils/dependency_file.py create mode 100644 tests/metagpt/utils/test_dependency_file.py diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..65d53364b 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,8 +11,9 @@ from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import PRDS_FILE_REPO, SYS_DESIGN_FILE_REPO, WORKSPACE_ROOT from metagpt.logs import logger +from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -202,7 +203,44 @@ async def _save(self, context, system_design): await self._save_prd(docs_path, resources_path, context) await self._save_system_design(docs_path, resources_path, system_design) - async def run(self, context, format=CONFIG.prompt_format): + async def run(self, with_messages, format=CONFIG.prompt_format): + # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + changed_prds = prds_file_repo.changed_files + # 通过git diff来识别docs/system_designs下那些设计文档发生了变动； + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYS_DESIGN_FILE_REPO) + changed_system_designs = system_design_file_repo.changed_files + + # 对于那些发生变动的PRD和设计文档，重新生成设计内容； + changed_files = Documents() + for filename in changed_prds.keys(): + prd = await prds_file_repo.get(filename) + old_system_design_doc = await system_design_file_repo.get(filename) + if not old_system_design_doc: + system_design = await self._run(context=prd.content) + doc = Document( + root_path=SYS_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + ) + else: + doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) + await system_design_file_repo.save( + filename=filename, content=doc.content, dependencies={prd.root_relative_path} + ) + changed_files.docs[filename] = doc + + for filename in changed_system_designs.keys(): + if filename in changed_files.docs: + continue + prd_doc = await prds_file_repo.get(filename=filename) + old_system_design_doc = await system_design_file_repo.get(filename) + new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) + await system_design_file_repo.save(filename=filename, content=new_system_design_doc.content) + changed_files.docs[filename] = new_system_design_doc + + # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + return ActionOutput(content=changed_files.json(), instruct_content=changed_files) + + async def _run(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) @@ -213,5 +251,8 @@ async def run(self, context, format=CONFIG.prompt_format): "Python package name", system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), ) - await self._save(context, system_design) + # await self._save(context, system_design) return system_design + + async def _merge(self, prd_doc, system_design_doc): + return system_design_doc diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b0185996b..c9b60ff27 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -7,19 +7,37 @@ @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ -from metagpt.actions import Action + +from pathlib import Path + +from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME, WORKSPACE_ROOT +from metagpt.schema import Document +from metagpt.utils.file_repository import FileRepository +from metagpt.utils.git_repository import GitRepository class PrepareDocuments(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, with_message, **kwargs): - parent = self.context.get("parent") - if not parent: - raise ValueError("Invalid owner") - env = parent.get_env() - if env.git_repository: - return - env.git_repository = GitRepository() - env.git_repository.open(WORKS) + async def run(self, with_messages, **kwargs): + if CONFIG.git_repo: + docs_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + doc = await docs_repo.get(REQUIREMENT_FILENAME) + return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) + + # Create and initialize the workspace folder, initialize the Git environment. + CONFIG.git_repo = GitRepository() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_file_name() + CONFIG.git_repo.open(local_path=workdir, auto_init=True) + + # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. + docs_file_repository = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) + await docs_file_repository.save(REQUIREMENT_FILENAME, content=doc.content) + + # Send a Message notification to the WritePRD action, instructing it to process requirements using + # `docs/requirement.txt` and `docs/prds/`. + return ActionOutput(content=doc.content, instruct_content=doc) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..a16d1ec06 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -10,7 +10,10 @@ from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.logs import logger +from metagpt.schema import Document, Documents +from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template templates = { @@ -222,7 +225,34 @@ class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + # 判断哪些需求文档需要重写：调LLM判断新增需求与prd是否相关，若相关就rewrite prd + docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + prd_docs = await prds_file_repo.get_all() + change_files = Documents() + for prd_doc in prd_docs: + if await self._is_relative_to(requirement_doc, prd_doc): + prd_doc = await self._merge(requirement_doc, prd_doc) + await prds_file_repo.save(filename=prd_doc.filename, content=prd_doc.content) + change_files.docs[prd_doc.filename] = prd_doc + # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd + if not change_files.docs: + prd = await self._run_new_requirement( + requirements=[requirement_doc.content], format=format, *args, **kwargs + ) + doc = Document( + root_path=PRDS_FILE_REPO, + filename=FileRepository.new_file_name() + ".json", + content=prd.instruct_content.json(), + ) + await prds_file_repo.save(filename=doc.filename, content=doc.content) + change_files.docs[doc.filename] = doc + # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 + return ActionOutput(content=change_files.json(), instruct_content=change_files) + + async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: sas = SearchAndSummarize() # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) rsp = "" @@ -239,3 +269,9 @@ async def run(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd + + async def _is_relative_to(self, doc1, doc2) -> bool: + return False + + async def _merge(self, doc1, doc2) -> Document: + pass diff --git a/metagpt/config.py b/metagpt/config.py index 27455d38d..51eed4fb8 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -46,7 +46,7 @@ def __init__(self, yaml_file=default_yaml_file): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -93,6 +93,7 @@ def __init__(self, yaml_file=default_yaml_file): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.git_repo = None def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/const.py b/metagpt/const.py index fa0ccc536..fc1c47b5b 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -49,3 +49,8 @@ def get_project_root(): MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" MESSAGE_ROUTE_TO_ALL = "" + +REQUIREMENT_FILENAME = "requirement.txt" +DOCS_FILE_REPO = "docs" +PRDS_FILE_REPO = "docs/prds" +SYS_DESIGN_FILE_REPO = "docs/system_design" diff --git a/metagpt/environment.py b/metagpt/environment.py index df93a818b..b3c296dac 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -12,7 +12,7 @@ functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable, Optional, Set +from typing import Iterable, Set from pydantic import BaseModel, Field @@ -20,7 +20,6 @@ from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import is_subscribed -from metagpt.utils.git_repository import GitRepository class Environment(BaseModel): @@ -32,7 +31,6 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) consumers: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug - git_repository: Optional[GitRepository] = None class Config: arbitrary_types_allowed = True @@ -113,8 +111,3 @@ def get_subscription(self, obj): def set_subscription(self, obj, tags): """Set the labels for message to be consumed by the object""" self.consumers[obj] = tags - - def dict(self, *args, **kwargs): - """Generate a dictionary representation of the model, optionally specifying which fields to include or - exclude.""" - return super(Environment, self).dict(exclude={"git_repository"}) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index c10aba6d1..81577ec2c 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -7,6 +7,7 @@ """ from metagpt.actions import BossRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG from metagpt.roles import Role @@ -38,12 +39,12 @@ def __init__( constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([PrepareDocuments(context={"parent": self}), WritePRD]) - self._watch([BossRequirement]) + self._init_actions([PrepareDocuments, WritePRD]) + self._watch([BossRequirement, PrepareDocuments]) async def _think(self) -> None: """Decide what to do""" - if self._rc.env.git_repository: + if CONFIG.git_repo: self._set_state(1) else: self._set_state(0) diff --git a/metagpt/schema.py b/metagpt/schema.py index 82a0117ef..674091e4c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -6,14 +6,16 @@ @File : schema.py @Modified By: mashenquan, 2023-10-31. According to Chapter 2.2.1 of RFC 116: Replanned the distribution of responsibilities and functional positioning of `Message` class attributes. +@Modified By: mashenquan, 2023/11/22. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. """ from __future__ import annotations import asyncio import json +import os.path from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError -from typing import List, Set, TypedDict +from typing import Dict, List, Optional, Set, TypedDict from pydantic import BaseModel, Field @@ -32,6 +34,42 @@ class RawMessage(TypedDict): role: str +class Document(BaseModel): + """ + Represents a document. + """ + + root_path: str + filename: str + content: Optional[str] = None + + def get_meta(self) -> Document: + """Get metadata of the document. + + :return: A new Document instance with the same root path and filename. + """ + + return Document(root_path=self.root_path, filename=self.filename) + + @property + def root_relative_path(self): + """Get relative path from root of git repository. + + :return: relative path from root of git repository. + """ + return os.path.join(self.root_path, self.filename) + + +class Documents(BaseModel): + """A class representing a collection of documents. + + Attributes: + docs (Dict[str, Document]): A dictionary mapping document names to Document instances. + """ + + docs: Dict[str, Document] = Field(default_factory=dict) + + class Message(BaseModel): """list[: ]""" diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py new file mode 100644 index 000000000..429027c7a --- /dev/null +++ b/metagpt/utils/dependency_file.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : dependency_file.py +@Desc: Implementation of the dependency file described in Section 2.2.3.2 of RFC 135. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Set + +import aiofiles + +from metagpt.logs import logger + + +class DependencyFile: + def __init__(self, workdir: Path | str): + self._dependencies = {} + self._filename = Path(workdir) / ".dependencies.json" + + async def load(self): + if not self._filename.exists(): + return + try: + async with aiofiles.open(str(self._filename), mode="r") as reader: + data = await reader.read() + self._dependencies = json.loads(data) + except Exception as e: + logger.error(f"Failed to load {str(self._filename)}, error:{e}") + + async def save(self): + try: + data = json.dumps(self._dependencies) + async with aiofiles.open(str(self._filename), mode="w") as writer: + await writer.write(data) + except Exception as e: + logger.error(f"Failed to save {str(self._filename)}, error:{e}") + + async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): + if persist: + await self.load() + + root = self._filename.parent + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + + if dependencies: + relative_paths = [] + for i in dependencies: + try: + relative_paths.append(str(Path(i).relative_to(root))) + except ValueError: + relative_paths.append(str(i)) + self._dependencies[str(key)] = relative_paths + elif str(key) in self._dependencies: + del self._dependencies[str(key)] + + if persist: + await self.save() + + async def get(self, filename: Path | str, persist=False): + if persist: + await self.load() + + root = self._filename.parent + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + return set(self._dependencies.get(str(key), {})) + + def delete_file(self): + self._filename.unlink(missing_ok=True) + + @property + def exists(self): + return self._filename.exists() diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index f4c36b5b7..7f07e4427 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -8,16 +8,29 @@ """ from __future__ import annotations -import json +import os +import uuid +from datetime import datetime from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Set import aiofiles from metagpt.logs import logger +from metagpt.schema import Document class FileRepository: + """A class representing a FileRepository associated with a Git repository. + + :param git_repo: The associated GitRepository instance. + :param relative_path: The relative path within the Git repository. + + Attributes: + _relative_path (Path): The relative path within the Git repository. + _git_repo (GitRepository): The associated GitRepository instance. + """ + def __init__(self, git_repo, relative_path: Path = Path(".")): """Initialize a FileRepository instance. @@ -26,16 +39,9 @@ def __init__(self, git_repo, relative_path: Path = Path(".")): """ self._relative_path = relative_path self._git_repo = git_repo - self._dependencies: Dict[str, List[str]] = {} # Initializing self.workdir.mkdir(parents=True, exist_ok=True) - if self.dependency_path_name.exists(): - try: - with open(str(self.dependency_path_name), mode="r") as reader: - self._dependencies = json.load(reader) - except Exception as e: - logger.error(f"Failed to load {str(self.dependency_path_name)}, error:{e}") async def save(self, filename: Path | str, content, dependencies: List[str] = None): """Save content to a file and update its dependencies. @@ -44,59 +50,68 @@ async def save(self, filename: Path | str, content, dependencies: List[str] = No :param content: The content to be saved. :param dependencies: List of dependency filenames or paths. """ - path_name = self.workdir / filename - path_name.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(str(path_name), mode="w") as writer: + pathname = self.workdir / filename + pathname.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(str(pathname), mode="w") as writer: await writer.write(content) - if dependencies is not None: - await self.update_dependency(filename, dependencies) - - async def get(self, filename: Path | str): - """Read the content of a file. + logger.info(f"save to: {str(pathname)}") - :param filename: The filename or path within the repository. - :return: The content of the file. - """ - path_name = self.workdir / filename - async with aiofiles.open(str(path_name), mode="r") as reader: - return await reader.read() + if dependencies is not None: + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(pathname, set(dependencies)) + logger.info(f"update dependency: {str(pathname)}:{dependencies}") - def get_dependency(self, filename: Path | str) -> List: + async def get_dependency(self, filename: Path | str) -> Set[str]: """Get the dependencies of a file. :param filename: The filename or path within the repository. - :return: List of dependency filenames or paths. + :return: Set of dependency filenames or paths. """ - key = str(filename) - return self._dependencies.get(key, []) + pathname = self.workdir / filename + dependency_file = await self._git_repo.get_dependency() + return await dependency_file.get(pathname) - def get_changed_dependency(self, filename: Path | str) -> List: + async def get_changed_dependency(self, filename: Path | str) -> Set[str]: """Get the dependencies of a file that have changed. :param filename: The filename or path within the repository. :return: List of changed dependency filenames or paths. """ - dependencies = self.get_dependency(filename=filename) + dependencies = await self.get_dependency(filename=filename) changed_files = self.changed_files - changed_dependent_files = [] + changed_dependent_files = set() for df in dependencies: if df in changed_files.keys(): - changed_dependent_files.append(df) + changed_dependent_files.add(df) return changed_dependent_files - async def update_dependency(self, filename, dependencies: List[str]): - """Update the dependencies of a file. + async def get(self, filename: Path | str) -> Document | None: + """Read the content of a file. :param filename: The filename or path within the repository. - :param dependencies: List of dependency filenames or paths. + :return: The content of the file. """ - self._dependencies[str(filename)] = dependencies + doc = Document(root_path=str(self.root_path), filename=str(filename)) + path_name = self.workdir / filename + if not path_name.exists(): + return None + async with aiofiles.open(str(path_name), mode="r") as reader: + doc.content = await reader.read() + return doc + + async def get_all(self) -> List[Document]: + """Get the content of all files in the repository. - async def save_dependency(self): - """Save the dependencies to a file.""" - data = json.dumps(self._dependencies) - with aiofiles.open(str(self.dependency_path_name), mode="w") as writer: - await writer.write(data) + :return: List of Document instances representing files. + """ + docs = [] + for root, dirs, files in os.walk(str(self.workdir)): + for file in files: + file_path = Path(root) / file + relative_path = file_path.relative_to(self.workdir) + doc = await self.get(relative_path) + docs.append(doc) + return docs @property def workdir(self): @@ -107,14 +122,9 @@ def workdir(self): return self._git_repo.workdir / self._relative_path @property - def dependency_path_name(self): - """Return the absolute path to the dependency file. - - :return: The absolute path to the dependency file. - """ - filename = ".dependencies.json" - path_name = self.workdir / filename - return path_name + def root_path(self): + """Return the relative path from git repository root""" + return self._relative_path @property def changed_files(self) -> Dict[str, str]: @@ -147,3 +157,13 @@ def get_change_dir_files(self, dir: Path | str) -> List: continue children.append(str(f)) return children + + @staticmethod + def new_file_name(): + """Generate a new filename based on the current timestamp and a UUID suffix. + + :return: A new filename string. + """ + current_time = datetime.now().strftime("%Y%m%d%H%M%S") + guid_suffix = str(uuid.uuid4())[:8] + return f"{current_time}t{guid_suffix}" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 6ae6a7900..a81b5c4ea 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -17,6 +17,7 @@ from git.repo.fun import is_git_dir from metagpt.const import WORKSPACE_ROOT +from metagpt.utils.dependency_file import DependencyFile from metagpt.utils.file_repository import FileRepository @@ -47,6 +48,7 @@ def __init__(self, local_path=None, auto_init=True): :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. """ self._repository = None + self._dependency = None if local_path: self.open(local_path=local_path, auto_init=auto_init) @@ -113,7 +115,7 @@ def is_git_dir(local_path): :param local_path: The local path to check. :return: True if the directory is a Git repository, False otherwise. """ - git_dir = local_path / ".git" + git_dir = Path(local_path) / ".git" if git_dir.exists() and is_git_dir(git_dir): return True return False @@ -151,7 +153,7 @@ def archive(self, comments="Archive"): self.add_change(self.changed_files) self.commit(comments) - def new_file_repository(self, relative_path: Path | str) -> FileRepository: + def new_file_repository(self, relative_path: Path | str = ".") -> FileRepository: """Create a new instance of FileRepository associated with this Git repository. :param relative_path: The relative path to the file repository within the Git repository. @@ -159,6 +161,15 @@ def new_file_repository(self, relative_path: Path | str) -> FileRepository: """ return FileRepository(git_repo=self, relative_path=Path(relative_path)) + async def get_dependency(self) -> DependencyFile: + """Get the dependency file associated with the Git repository. + + :return: An instance of DependencyFile. + """ + if not self._dependency: + self._dependency = DependencyFile(workdir=self.workdir) + return self._dependency + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" diff --git a/requirements.txt b/requirements.txt index c3b909e77..73a03d537 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,4 +44,4 @@ ta==0.10.2 semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 - +aiofiles==23.2.1 diff --git a/startup.py b/startup.py index e2a903c9b..d5a6bb07b 100644 --- a/startup.py +++ b/startup.py @@ -4,6 +4,7 @@ import fire +from metagpt.config import CONFIG from metagpt.roles import ( Architect, Engineer, @@ -54,6 +55,7 @@ def main( code_review: bool = True, run_tests: bool = False, implement: bool = True, + project_path: str = None, ): """ We are a software startup comprised of AI. By investing in us, @@ -63,8 +65,12 @@ def main( a certain dollar amount to this AI company. :param n_round: :param code_review: Whether to use code review. + :param run_tests: Whether run unit tests. + :param implement: Whether to write codes. + :param project_path: The path of the old version project to improve. :return: """ + CONFIG.WORKDIR = project_path asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) diff --git a/tests/metagpt/utils/test_dependency_file.py b/tests/metagpt/utils/test_dependency_file.py new file mode 100644 index 000000000..ae4d40ea5 --- /dev/null +++ b/tests/metagpt/utils/test_dependency_file.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : test_dependency_file.py +@Desc: Unit tests for dependency_file.py +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Set, Union + +import pytest +from pydantic import BaseModel + +from metagpt.utils.dependency_file import DependencyFile + + +@pytest.mark.asyncio +async def test_dependency_file(): + class Input(BaseModel): + x: Union[Path, str] + deps: Optional[Set[Union[Path, str]]] + key: Optional[Union[Path, str]] + want: Set[str] + + inputs = [ + Input(x="a/b.txt", deps={"c/e.txt", Path(__file__).parent / "d.txt"}, want={"c/e.txt", "d.txt"}), + Input( + x=Path(__file__).parent / "x/b.txt", + deps={"s/e.txt", Path(__file__).parent / "d.txt"}, + key="x/b.txt", + want={"s/e.txt", "d.txt"}, + ), + Input(x="f.txt", deps=None, want=set()), + Input(x="a/b.txt", deps=None, want=set()), + ] + + file = DependencyFile(workdir=Path(__file__).parent) + + for i in inputs: + await file.update(filename=i.x, dependencies=i.deps) + assert await file.get(filename=i.key or i.x) == i.want + + file2 = DependencyFile(workdir=Path(__file__).parent) + file2.delete_file() + assert not file.exists + await file2.update(filename="a/b.txt", dependencies={"c/e.txt", Path(__file__).parent / "d.txt"}, persist=False) + assert not file.exists + await file2.save() + assert file2.exists + + file1 = DependencyFile(workdir=Path(__file__).parent) + assert file1.exists + assert await file1.get("a/b.txt") == set() + await file1.load() + assert await file1.get("a/b.txt") == {"c/e.txt", "d.txt"} + file1.delete_file() + assert not file.exists + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py index ac36f2320..a830b58aa 100644 --- a/tests/metagpt/utils/test_file_repository.py +++ b/tests/metagpt/utils/test_file_repository.py @@ -34,11 +34,13 @@ async def test_file_repo(): assert file_repo.workdir.exists() await file_repo.save("a.txt", "AAA") await file_repo.save("b.txt", "BBB", ["a.txt"]) - assert "AAA" == await file_repo.get("a.txt") - assert "BBB" == await file_repo.get("b.txt") - assert ["a.txt"] == file_repo.get_dependency("b.txt") + doc = await file_repo.get("a.txt") + assert "AAA" == doc.content + doc = await file_repo.get("b.txt") + assert "BBB" == doc.content + assert {"a.txt"} == await file_repo.get_dependency("b.txt") assert {"a.txt": ChangeType.UNTRACTED, "b.txt": ChangeType.UNTRACTED} == file_repo.changed_files - assert ["a.txt"] == file_repo.get_changed_dependency("b.txt") + assert {"a.txt"} == await file_repo.get_changed_dependency("b.txt") await file_repo.save("d/e.txt", "EEE") assert ["d/e.txt"] == file_repo.get_change_dir_files("d") diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index 0d1e3b791..23bebba7f 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -77,5 +77,20 @@ async def test_git1(): assert not local_path.exists() +@pytest.mark.asyncio +async def test_dependency_file(): + local_path = Path(__file__).parent / "git2" + repo, subdir = await mock_repo(local_path) + + dependancy_file = await repo.get_dependency() + assert not dependancy_file.exists + + await dependancy_file.update(filename="a/b.txt", dependencies={"c/d.txt", "e/f.txt"}) + assert dependancy_file.exists + + repo.delete_repository() + assert not dependancy_file.exists + + if __name__ == "__main__": pytest.main([__file__, "-s"]) From 9339eab20c95263549c8ad60a6bad087ab2cac46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 20:40:42 +0800 Subject: [PATCH 124/232] feat: archive --- metagpt/actions/design_api.py | 22 +++++++--- metagpt/actions/project_management.py | 63 ++++++++++++++++++++++++++- metagpt/const.py | 3 +- metagpt/utils/file_repository.py | 2 +- metagpt/utils/git_repository.py | 16 +++++++ 5 files changed, 97 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 65d53364b..e7ee87fa2 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,7 @@ from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import PRDS_FILE_REPO, SYS_DESIGN_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import PRDS_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser @@ -208,7 +208,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format): prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) changed_prds = prds_file_repo.changed_files # 通过git diff来识别docs/system_designs下那些设计文档发生了变动； - system_design_file_repo = CONFIG.git_repo.new_file_repository(SYS_DESIGN_FILE_REPO) + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files # 对于那些发生变动的PRD和设计文档，重新生成设计内容； @@ -219,7 +219,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format): if not old_system_design_doc: system_design = await self._run(context=prd.content) doc = Document( - root_path=SYS_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() ) else: doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) @@ -234,7 +234,9 @@ async def run(self, with_messages, format=CONFIG.prompt_format): prd_doc = await prds_file_repo.get(filename=filename) old_system_design_doc = await system_design_file_repo.get(filename) new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) - await system_design_file_repo.save(filename=filename, content=new_system_design_doc.content) + await system_design_file_repo.save( + filename=filename, content=new_system_design_doc.content, dependencies={prd_doc.root_relative_path} + ) changed_files.docs[filename] = new_system_design_doc # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 @@ -251,8 +253,18 @@ async def _run(self, context, format=CONFIG.prompt_format): "Python package name", system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), ) - # await self._save(context, system_design) + await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc): return system_design_doc + + async def _rename_workspace(self, system_design): + if CONFIG.WORKDIR: # 已经指定了在旧版本上更新 + return + + if isinstance(system_design, ActionOutput): + ws_name = system_design.instruct_content.dict()["Python package name"] + else: + ws_name = CodeParser.parse_str(block="Python package name", text=system_design) + CONFIG.git_repo.rename_root(ws_name) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b395fa64e..73481c780 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -5,11 +5,14 @@ @Author : alexanderwu @File : project_management.py """ +import json from typing import List +from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, WORKSPACE_ROOT +from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -178,13 +181,69 @@ def _save(self, context, rsp): requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) - async def run(self, context, format=CONFIG.prompt_format): + async def run(self, with_messages, format=CONFIG.prompt_format): + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) + changed_system_designs = system_design_file_repo.changed_files + + tasks_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) + changed_tasks = tasks_file_repo.changed_files + change_files = Documents() + # 根据docs/system_designs/下的git head diff识别哪些task文档需要重写 + for filename in changed_system_designs: + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + if task_doc: + task_doc = await self._merge(system_design_doc, task_doc) + else: + rsp = await self._run(system_design_doc.content) + task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + change_files.docs[filename] = task_doc + + # 根据docs/tasks/下的git head diff识别哪些task文件被用户修改了，需要重写 + for filename in changed_tasks: + if filename in change_files.docs: + continue + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + task_doc = await self._merge(system_design_doc, task_doc) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + change_files.docs[filename] = task_doc + + # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + return ActionOutput(content=change_files.json(), instruct_content=change_files) + + async def _run(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) self._save(context, rsp) return rsp + async def _merge(self, system_design_doc, task_dock) -> Document: + return task_dock + + async def _update_requirements(self, doc): + m = json.loads(doc.content) + packages = set(m.get("Required Python third-party packages", set())) + file_repo = CONFIG.git_repo.new_file_repository() + filename = "requirements.txt" + requirement_doc = await file_repo.get(filename) + if not requirement_doc: + requirement_doc = Document(filename=filename, root_path=".", content="") + lines = requirement_doc.content.splitlines() + for pkg in lines: + if pkg == "": + continue + packages.add(pkg) + await file_repo.save(filename, content="\n".join(packages)) + class AssignTasks(Action): async def run(self, *args, **kwargs): diff --git a/metagpt/const.py b/metagpt/const.py index fc1c47b5b..63f39f4a8 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -53,4 +53,5 @@ def get_project_root(): REQUIREMENT_FILENAME = "requirement.txt" DOCS_FILE_REPO = "docs" PRDS_FILE_REPO = "docs/prds" -SYS_DESIGN_FILE_REPO = "docs/system_design" +SYSTEM_DESIGN_FILE_REPO = "docs/system_design" +TASK_FILE_REPO = "docs/tasks" diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 7f07e4427..ee6811209 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -166,4 +166,4 @@ def new_file_name(): """ current_time = datetime.now().strftime("%Y%m%d%H%M%S") guid_suffix = str(uuid.uuid4())[:8] - return f"{current_time}t{guid_suffix}" + return f"{current_time}x{guid_suffix}" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index a81b5c4ea..2a4fb4a4d 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -17,6 +17,7 @@ from git.repo.fun import is_git_dir from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile from metagpt.utils.file_repository import FileRepository @@ -170,6 +171,21 @@ async def get_dependency(self) -> DependencyFile: self._dependency = DependencyFile(workdir=self.workdir) return self._dependency + def rename_root(self, new_dir_name): + """Rename the root directory of the Git repository. + + :param new_dir_name: The new name for the root directory. + """ + if self.workdir.name == new_dir_name: + return + new_path = self.workdir.parent / new_dir_name + if new_path.exists(): + logger.info(f"Delete directory {str(new_path)}") + shutil.rmtree(new_path) + self.workdir.rename(new_path) + logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") + self._repository = Repo(new_path) + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" From e8131652de02a93454343d059dec02199f27b459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 21:45:44 +0800 Subject: [PATCH 125/232] refactor: write prd & system design --- metagpt/actions/design_api.py | 83 ++++++++++++++++++++++++++--------- metagpt/actions/write_prd.py | 70 ++++++++++++++++++++++------- metagpt/const.py | 5 +++ 3 files changed, 123 insertions(+), 35 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index e7ee87fa2..3bbde24ea 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -5,13 +5,21 @@ @Author : alexanderwu @File : design_api.py """ +import json import shutil from pathlib import Path from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import PRDS_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import ( + DATA_API_DESIGN_FILE_REPO, + PRDS_FILE_REPO, + SEQ_FLOW_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + SYSTEM_DESIGN_PDF_FILE_REPO, + WORKSPACE_ROOT, +) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser @@ -214,40 +222,29 @@ async def run(self, with_messages, format=CONFIG.prompt_format): # 对于那些发生变动的PRD和设计文档，重新生成设计内容； changed_files = Documents() for filename in changed_prds.keys(): - prd = await prds_file_repo.get(filename) - old_system_design_doc = await system_design_file_repo.get(filename) - if not old_system_design_doc: - system_design = await self._run(context=prd.content) - doc = Document( - root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() - ) - else: - doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) - await system_design_file_repo.save( - filename=filename, content=doc.content, dependencies={prd.root_relative_path} + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) changed_files.docs[filename] = doc for filename in changed_system_designs.keys(): if filename in changed_files.docs: continue - prd_doc = await prds_file_repo.get(filename=filename) - old_system_design_doc = await system_design_file_repo.get(filename) - new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) - await system_design_file_repo.save( - filename=filename, content=new_system_design_doc.content, dependencies={prd_doc.root_relative_path} + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) - changed_files.docs[filename] = new_system_design_doc + changed_files.docs[filename] = doc # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - async def _run(self, context, format=CONFIG.prompt_format): + async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr + # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python + # package name" contain space, have to use setattr setattr( system_design.instruct_content, "Python package name", @@ -268,3 +265,49 @@ async def _rename_workspace(self, system_design): else: ws_name = CodeParser.parse_str(block="Python package name", text=system_design) CONFIG.git_repo.rename_root(ws_name) + + async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: + prd = await prds_file_repo.get(filename) + old_system_design_doc = await system_design_file_repo.get(filename) + if not old_system_design_doc: + system_design = await self._new_system_design(context=prd.content) + doc = Document( + root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + ) + else: + doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) + await system_design_file_repo.save( + filename=filename, content=doc.content, dependencies={prd.root_relative_path} + ) + await self._save_data_api_design(doc) + await self._save_seq_flow(doc) + await self._save_pdf(doc) + return doc + + @staticmethod + async def _save_data_api_design(design_doc): + m = json.loads(design_doc.content) + data_api_design = m.get("Data structures and interface definitions") + if not data_api_design: + return + path = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(data_api_design, path / Path(design_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_seq_flow(design_doc): + m = json.loads(design_doc.content) + seq_flow = m.get("Program call flow") + if not seq_flow: + return + path = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(seq_flow, path / Path(design_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_pdf(design_doc): + m = json.loads(design_doc.content) + file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) + await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index a16d1ec06..df35ec865 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -5,16 +5,28 @@ @Author : alexanderwu @File : write_prd.py """ +from __future__ import annotations + +import json +from pathlib import Path from typing import List from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG -from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.const import ( + COMPETITIVE_ANALYSIS_FILE_REPO, + DOCS_FILE_REPO, + PRD_PDF_FILE_REPO, + PRDS_FILE_REPO, + REQUIREMENT_FILENAME, +) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template +from metagpt.utils.json_to_markdown import json_to_markdown +from metagpt.utils.mermaid import mermaid_to_file templates = { "json": { @@ -233,22 +245,15 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) prd_docs = await prds_file_repo.get_all() change_files = Documents() for prd_doc in prd_docs: - if await self._is_relative_to(requirement_doc, prd_doc): - prd_doc = await self._merge(requirement_doc, prd_doc) - await prds_file_repo.save(filename=prd_doc.filename, content=prd_doc.content) - change_files.docs[prd_doc.filename] = prd_doc + prd_doc = await self._update_prd(requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) + if not prd_doc: + continue + change_files.docs[prd_doc.filename] = prd_doc # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd if not change_files.docs: - prd = await self._run_new_requirement( - requirements=[requirement_doc.content], format=format, *args, **kwargs - ) - doc = Document( - root_path=PRDS_FILE_REPO, - filename=FileRepository.new_file_name() + ".json", - content=prd.instruct_content.json(), - ) - await prds_file_repo.save(filename=doc.filename, content=doc.content) - change_files.docs[doc.filename] = doc + prd_doc = await self._update_prd(requirement_doc, None, prds_file_repo) + if prd_doc: + change_files.docs[prd_doc.filename] = prd_doc # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) @@ -275,3 +280,38 @@ async def _is_relative_to(self, doc1, doc2) -> bool: async def _merge(self, doc1, doc2) -> Document: pass + + async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: + if not prd_doc: + prd = await self._run_new_requirement( + requirements=[requirement_doc.content], format=format, *args, **kwargs + ) + new_prd_doc = Document( + root_path=PRDS_FILE_REPO, + filename=FileRepository.new_file_name() + ".json", + content=prd.instruct_content.json(), + ) + elif await self._is_relative_to(requirement_doc, prd_doc): + new_prd_doc = await self._merge(requirement_doc, prd_doc) + else: + return None + await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) + await self._save_competitive_analysis(new_prd_doc) + await self._save_pdf(new_prd_doc) + + @staticmethod + async def _save_competitive_analysis(prd_doc): + m = json.loads(prd_doc.content) + quadrant_chart = m.get("Competitive Quadrant Chart") + if not quadrant_chart: + return + path = CONFIG.git_repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(quadrant_chart, path / Path(prd_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_pdf(prd_doc): + m = json.loads(prd_doc.content) + file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) + await file_repo.save(filename=prd_doc.filename, content=json_to_markdown(m)) diff --git a/metagpt/const.py b/metagpt/const.py index 63f39f4a8..b5ecad7cc 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -55,3 +55,8 @@ def get_project_root(): PRDS_FILE_REPO = "docs/prds" SYSTEM_DESIGN_FILE_REPO = "docs/system_design" TASK_FILE_REPO = "docs/tasks" +COMPETITIVE_ANALYSIS_FILE_REPO = "resources/competitive_analysis" +DATA_API_DESIGN_FILE_REPO = "resources/data_api_design" +SEQ_FLOW_FILE_REPO = "resources/seq_flow" +SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" +PRD_PDF_FILE_REPO = "resources/prd" From 62d93517b48824c40209e447e8f76a59a7744d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 21:59:14 +0800 Subject: [PATCH 126/232] refactor: write prd & system design --- metagpt/actions/design_api.py | 21 +++++++++++++-------- metagpt/actions/write_prd.py | 8 ++++---- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 3bbde24ea..8fb926477 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -290,10 +290,9 @@ async def _save_data_api_design(design_doc): data_api_design = m.get("Data structures and interface definitions") if not data_api_design: return - path = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO - if not path.exists(): - path.mkdir(parents=True, exists_ok=True) - await mermaid_to_file(data_api_design, path / Path(design_doc).with_suffix(".mmd")) + pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc).with_suffix(".mmd") + await WriteDesign._save_mermaid_file(data_api_design, pathname) + logger.info(f"Save class view to {str(pathname)}") @staticmethod async def _save_seq_flow(design_doc): @@ -301,13 +300,19 @@ async def _save_seq_flow(design_doc): seq_flow = m.get("Program call flow") if not seq_flow: return - path = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO - if not path.exists(): - path.mkdir(parents=True, exists_ok=True) - await mermaid_to_file(seq_flow, path / Path(design_doc).with_suffix(".mmd")) + pathname = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO / Path(design_doc).with_suffix(".mmd") + await WriteDesign._save_mermaid_file(seq_flow, pathname) + logger.info(f"Saving sequence flow to {str(pathname)}") @staticmethod async def _save_pdf(design_doc): m = json.loads(design_doc.content) file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) + logger.info(f"Saving system design pdf to {design_doc.root_relative_path}") + + @staticmethod + async def _save_mermaid_file(data: str, pathname: Path): + if not pathname.parent.exists(): + pathname.parent.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(data, pathname) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index df35ec865..34001dec1 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -305,10 +305,10 @@ async def _save_competitive_analysis(prd_doc): quadrant_chart = m.get("Competitive Quadrant Chart") if not quadrant_chart: return - path = CONFIG.git_repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO - if not path.exists(): - path.mkdir(parents=True, exists_ok=True) - await mermaid_to_file(quadrant_chart, path / Path(prd_doc).with_suffix(".mmd")) + pathname = CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc).with_suffix(".mmd") + if not pathname.parent.exists(): + pathname.parent.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(quadrant_chart, pathname) @staticmethod async def _save_pdf(prd_doc): From 369047e5586ef52cd21b9bc401630dfbba23fa29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 22:00:51 +0800 Subject: [PATCH 127/232] refactor: write prd & system design --- metagpt/actions/design_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 8fb926477..2c8c87558 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -290,7 +290,7 @@ async def _save_data_api_design(design_doc): data_api_design = m.get("Data structures and interface definitions") if not data_api_design: return - pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc).with_suffix(".mmd") await WriteDesign._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") @@ -300,7 +300,7 @@ async def _save_seq_flow(design_doc): seq_flow = m.get("Program call flow") if not seq_flow: return - pathname = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc).with_suffix(".mmd") await WriteDesign._save_mermaid_file(seq_flow, pathname) logger.info(f"Saving sequence flow to {str(pathname)}") From 438fbe28c06aefee542bdd005941f801d4fe3e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 11:29:09 +0800 Subject: [PATCH 128/232] refactor: save files --- metagpt/actions/design_api.py | 11 ++---- metagpt/actions/project_management.py | 54 +++++++++++++++++---------- metagpt/actions/write_prd.py | 23 +++++++----- metagpt/const.py | 1 + metagpt/utils/file_repository.py | 18 +++++++-- 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 2c8c87558..a8f89473d 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -290,7 +290,7 @@ async def _save_data_api_design(design_doc): data_api_design = m.get("Data structures and interface definitions") if not data_api_design: return - pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") @@ -300,19 +300,16 @@ async def _save_seq_flow(design_doc): seq_flow = m.get("Program call flow") if not seq_flow: return - pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(seq_flow, pathname) logger.info(f"Saving sequence flow to {str(pathname)}") @staticmethod async def _save_pdf(design_doc): - m = json.loads(design_doc.content) file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) - await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) - logger.info(f"Saving system design pdf to {design_doc.root_relative_path}") + await file_repo.save_pdf(doc=design_doc) @staticmethod async def _save_mermaid_file(data: str, pathname: Path): - if not pathname.parent.exists(): - pathname.parent.mkdir(parents=True, exists_ok=True) + pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(data, pathname) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 73481c780..686aa3689 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -11,7 +11,12 @@ from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import ( + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, + TASK_PDF_FILE_REPO, + WORKSPACE_ROOT, +) from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -190,46 +195,50 @@ async def run(self, with_messages, format=CONFIG.prompt_format): change_files = Documents() # 根据docs/system_designs/下的git head diff识别哪些task文档需要重写 for filename in changed_system_designs: - system_design_doc = await system_design_file_repo.get(filename) - task_doc = await tasks_file_repo.get(filename) - if task_doc: - task_doc = await self._merge(system_design_doc, task_doc) - else: - rsp = await self._run(system_design_doc.content) - task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) - await tasks_file_repo.save( - filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) - await self._update_requirements(task_doc) change_files.docs[filename] = task_doc # 根据docs/tasks/下的git head diff识别哪些task文件被用户修改了，需要重写 for filename in changed_tasks: if filename in change_files.docs: continue - system_design_doc = await system_design_file_repo.get(filename) - task_doc = await tasks_file_repo.get(filename) - task_doc = await self._merge(system_design_doc, task_doc) - await tasks_file_repo.save( - filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) - await self._update_requirements(task_doc) change_files.docs[filename] = task_doc # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) - async def _run(self, context, format=CONFIG.prompt_format): + async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo): + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + if task_doc: + task_doc = await self._merge(system_design_doc=system_design_doc, task_dock=task_doc) + else: + rsp = await self._run_new_tasks(context=system_design_doc.content) + task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + await self._save_pdf(task_doc=task_doc) + return task_doc + + async def _run_new_tasks(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - self._save(context, rsp) + # self._save(context, rsp) return rsp async def _merge(self, system_design_doc, task_dock) -> Document: return task_dock - async def _update_requirements(self, doc): + @staticmethod + async def _update_requirements(doc): m = json.loads(doc.content) packages = set(m.get("Required Python third-party packages", set())) file_repo = CONFIG.git_repo.new_file_repository() @@ -244,6 +253,11 @@ async def _update_requirements(self, doc): packages.add(pkg) await file_repo.save(filename, content="\n".join(packages)) + @staticmethod + async def _save_pdf(task_doc): + file_repo = CONFIG.git_repo.new_file_repository(TASK_PDF_FILE_REPO) + await file_repo.save_pdf(doc=task_doc) + class AssignTasks(Action): async def run(self, *args, **kwargs): diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 34001dec1..8b03ac29a 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -25,7 +25,6 @@ from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -245,13 +244,17 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) prd_docs = await prds_file_repo.get_all() change_files = Documents() for prd_doc in prd_docs: - prd_doc = await self._update_prd(requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=prd_doc, prds_file_repo=prds_file_repo, *args, **kwargs + ) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd if not change_files.docs: - prd_doc = await self._update_prd(requirement_doc, None, prds_file_repo) + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs + ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 @@ -283,9 +286,7 @@ async def _merge(self, doc1, doc2) -> Document: async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: - prd = await self._run_new_requirement( - requirements=[requirement_doc.content], format=format, *args, **kwargs - ) + prd = await self._run_new_requirement(requirements=[requirement_doc.content], *args, **kwargs) new_prd_doc = Document( root_path=PRDS_FILE_REPO, filename=FileRepository.new_file_name() + ".json", @@ -298,6 +299,7 @@ async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **k await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) await self._save_competitive_analysis(new_prd_doc) await self._save_pdf(new_prd_doc) + return new_prd_doc @staticmethod async def _save_competitive_analysis(prd_doc): @@ -305,13 +307,14 @@ async def _save_competitive_analysis(prd_doc): quadrant_chart = m.get("Competitive Quadrant Chart") if not quadrant_chart: return - pathname = CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc).with_suffix(".mmd") + pathname = ( + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + ) if not pathname.parent.exists(): - pathname.parent.mkdir(parents=True, exists_ok=True) + pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(quadrant_chart, pathname) @staticmethod async def _save_pdf(prd_doc): - m = json.loads(prd_doc.content) file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) - await file_repo.save(filename=prd_doc.filename, content=json_to_markdown(m)) + await file_repo.save_pdf(doc=prd_doc) diff --git a/metagpt/const.py b/metagpt/const.py index b5ecad7cc..7ee06ff7d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -60,3 +60,4 @@ def get_project_root(): SEQ_FLOW_FILE_REPO = "resources/seq_flow" SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" +TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index ee6811209..62ba99d42 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -8,8 +8,8 @@ """ from __future__ import annotations +import json import os -import uuid from datetime import datetime from pathlib import Path from typing import Dict, List, Set @@ -18,6 +18,7 @@ from metagpt.logs import logger from metagpt.schema import Document +from metagpt.utils.json_to_markdown import json_to_markdown class FileRepository: @@ -165,5 +166,16 @@ def new_file_name(): :return: A new filename string. """ current_time = datetime.now().strftime("%Y%m%d%H%M%S") - guid_suffix = str(uuid.uuid4())[:8] - return f"{current_time}x{guid_suffix}" + return current_time + # guid_suffix = str(uuid.uuid4())[:8] + # return f"{current_time}x{guid_suffix}" + + async def save_pdf(self, doc: Document): + """Save a Document as a PDF file. + + :param doc: The Document instance to be saved. + """ + m = json.loads(doc.content) + filename = Path(doc.filename).with_suffix(".md") + await self.save(filename=str(filename), content=json_to_markdown(m)) + logger.info(f"File Saved: {str(filename)}") From 0fdeab3f200f4a5cdb0b6c8427373f4f037d6ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 17:36:01 +0800 Subject: [PATCH 129/232] fixbug: Message was incorrectly filtered by the profile. --- metagpt/roles/qa_engineer.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 760b65736..b57b64a7e 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -151,13 +151,6 @@ async def _debug_error(self, msg): ) self.publish_message(msg) - async def _observe(self) -> int: - await super()._observe() - self._rc.news = [ - msg for msg in self._rc.news if self.profile in msg.send_to - ] # only relevant msgs count as observed news - return len(self._rc.news) - async def _act(self) -> Message: if self.test_round > self.test_round_allowed: result_msg = Message( From 2032a385426e44fea0154a292aa8e4b1e1a9be59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 17:49:38 +0800 Subject: [PATCH 130/232] feat: rewrite Engineer & WriteCode & WriteCodeReview --- metagpt/actions/write_code.py | 24 ++- metagpt/actions/write_code_review.py | 25 ++- metagpt/config.py | 1 + metagpt/roles/engineer.py | 272 +++++++++++---------------- metagpt/roles/qa_engineer.py | 7 - metagpt/schema.py | 7 + startup.py | 2 + 7 files changed, 155 insertions(+), 183 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index aeaa10aec..d4d33fe0c 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,13 +7,15 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. """ +import json + from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions import WriteDesign from metagpt.actions.action import Action from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.schema import Message +from metagpt.schema import CodingContext from metagpt.utils.common import CodeParser, any_to_str PROMPT_TEMPLATE = """ @@ -46,7 +48,7 @@ class WriteCode(Action): - def __init__(self, name="WriteCode", context: list[Message] = None, llm=None): + def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) def _is_invalid(self, filename): @@ -70,15 +72,19 @@ def _save(self, context, filename, code): logger.info(f"Saving Code to {code_path}") @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code(self, prompt): + async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=code_rsp) return code - async def run(self, context, filename): - prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f"Writing {filename}..") + async def run(self, *args, **kwargs) -> CodingContext: + m = json.loads(self.context.content) + coding_context = CodingContext(**m) + context = "\n".join( + [coding_context.design_doc.content, coding_context.task_doc.content, coding_context.code_doc.content] + ) + prompt = PROMPT_TEMPLATE.format(context=context, filename=self.context.filename) + logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) - # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) - # self._save(context, filename, code) - return code + coding_context.code_doc.content = code + return coding_context diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 76adca255..10e4aec3b 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -10,7 +10,7 @@ from metagpt.actions.action import Action from metagpt.logs import logger -from metagpt.schema import Message +from metagpt.schema import CodingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -63,7 +63,7 @@ class WriteCodeReview(Action): - def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None): + def __init__(self, name="WriteCodeReview", context=None, llm=None): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) @@ -72,11 +72,18 @@ async def write_code(self, prompt): code = CodeParser.parse_code(block="", text=code_rsp) return code - async def run(self, context, code, filename): - format_example = FORMAT_EXAMPLE.format(filename=filename) - prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) - logger.info(f"Code review {filename}..") + async def run(self, *args, **kwargs) -> CodingContext: + format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) + context = "\n".join( + [self.context.design_doc.content, self.context.task_doc.content, self.context.code_doc.content] + ) + prompt = PROMPT_TEMPLATE.format( + context=context, + code=self.context.code_doc.content, + filename=self.context.code_doc.filename, + format_example=format_example, + ) + logger.info(f"Code review {self.context.code_doc.filename}..") code = await self.write_code(prompt) - # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) - # self._save(context, filename, code) - return code + self.context.code_doc.content = code + return self.context diff --git a/metagpt/config.py b/metagpt/config.py index 51eed4fb8..d059a6a29 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -94,6 +94,7 @@ def __init__(self, yaml_file=default_yaml_file): self.prompt_format = self._get("PROMPT_FORMAT", "markdown") self.git_repo = None + self.src_workspace = None def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d23d23d55..8852d55f1 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,47 +11,20 @@ 3. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. 4. Supplemented the external transmission of internal messages. """ -import asyncio -import shutil -from collections import OrderedDict +from __future__ import annotations + +import json from pathlib import Path -from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks +from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser, any_to_str +from metagpt.schema import CodingContext, Document, Documents, Message from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP -async def gather_ordered_k(coros, k) -> list: - tasks = OrderedDict() - results = [None] * len(coros) - done_queue = asyncio.Queue() - - for i, coro in enumerate(coros): - if len(tasks) >= k: - done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED) - for task in done: - index = tasks.pop(task) - await done_queue.put((index, task.result())) - task = asyncio.create_task(coro) - tasks[task] = i - - if tasks: - done, _ = await asyncio.wait(tasks.keys()) - for task in done: - index = tasks[task] - await done_queue.put((index, task.result())) - - while not done_queue.empty(): - index, result = await done_queue.get() - results[index] = result - - return results - - class Engineer(Role): """ Represents an Engineer role responsible for writing and possibly reviewing code. @@ -77,105 +50,19 @@ def __init__( ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) - self._init_actions([WriteCode]) self.use_code_review = use_code_review - if self.use_code_review: - self._init_actions([WriteCode, WriteCodeReview]) self._watch([WriteTasks]) self.todos = [] self.n_borg = n_borg - @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: - if task_msg.instruct_content: - return task_msg.instruct_content.dict().get("Task list") - return CodeParser.parse_file_list(block="Task list", text=task_msg.content) - - @classmethod - def parse_code(self, code_text: str) -> str: - return CodeParser.parse_code(block="", text=code_text) - - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - - def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return WORKSPACE_ROOT / "src" - workspace = self.parse_workspace(msg) - # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace - - def recreate_workspace(self): - workspace = self.get_workspace() - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # The folder does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - filename = filename.replace('"', "").replace("\n", "") - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - return file - - async def _act_mp(self) -> Message: - # self.recreate_workspace() - todo_coros = [] - for todo in self.todos: - todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), - filename=todo, - ) - todo_coros.append(todo_coro) - - rsps = await gather_ordered_k(todo_coros, self.n_borg) - for todo, code_rsp in zip(self.todos, rsps): - _ = self.parse_code(code_rsp) - logger.info(todo) - logger.info(code_rsp) - # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=self._rc.todo) - self._rc.memory.add(msg) - self.publish_message(msg) - del self.todos[0] - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message(content="all done.", role=self.profile, cause_by=self._rc.todo) - return msg - - async def _act_sp(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later - for todo in self.todos: - code = await WriteCode().run(context=self._rc.history, filename=todo) - # logger.info(todo) - # logger.info(code_rsp) - # code = self.parse_code(code_rsp) - file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=self._rc.todo) - self._rc.memory.add(msg) - self.publish_message(msg) - - code_msg = todo + FILENAME_CODE_SEP + str(file_path) - code_msg_all.append(code_msg) + @staticmethod + def _parse_tasks(task_msg: Document) -> list[str]: + m = json.loads(task_msg.content) + return m.get("Task list") - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message( - content=MSG_SEP.join(code_msg_all), - role=self.profile, - cause_by=self._rc.todo, - send_to="Edward", - ) - return msg - - async def _act_sp_precision(self) -> Message: + async def _act_sp_precision(self, review=False) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) for todo in self.todos: """ # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): @@ -184,30 +71,29 @@ async def _act_sp_precision(self) -> Message: 3. Do we need other codes (currently needed)? TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ - context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - # Write code - code = await WriteCode().run(context=context_str, filename=todo) + coding_context = await todo.run() # Code review - if self.use_code_review: + if review: try: - rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - code = rewrite_code + coding_context = await WriteCodeReview(context=coding_context, llm=self._llm).run() except Exception as e: logger.error("code review failed!", e) pass - file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) + await src_file_repo.save( + coding_context.filename, + dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}, + content=coding_context.code_doc.content, + ) + msg = Message( + content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode + ) self._rc.memory.add(msg) self.publish_message(msg) - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = coding_context.filename + FILENAME_CODE_SEP + str(coding_context.code_doc.root_relative_path) code_msg_all.append(code_msg) - logger.info(f"Done {self.get_workspace()} generating.") + logger.info(f"Done {CONFIG.src_workspace} generating.") msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, @@ -218,22 +104,92 @@ async def _act_sp_precision(self) -> Message: async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" - if not self._rc.todo: - return None - if self.use_code_review: - return await self._act_sp_precision() - return await self._act_sp() - - async def _observe(self) -> int: - ret = await super(Engineer, self)._observe() - if ret == 0: - return ret - - # Parse task lists - for message in self._rc.news: - if not message.cause_by == any_to_str(WriteTasks): + return await self._act_sp_precision(review=self.use_code_review) + + async def _think(self) -> Action | None: + if not CONFIG.src_workspace: + CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name + # Prepare file repos + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_src_files = src_file_repo.changed_files + task_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) + changed_task_files = task_file_repo.changed_files + design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) + + changed_files = Documents() + # 由上游变化导致的recode + for filename in changed_task_files: + design_doc = await design_file_repo.get(filename) + task_doc = await task_file_repo.get(filename) + task_list = self._parse_tasks(task_doc) + for task_filename in task_list: + old_code_doc = await src_file_repo.get(task_filename) + if not old_code_doc: + old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=task_filename, content="") + context = CodingContext( + filename=task_filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc + ) + coding_doc = Document( + root_path=str(src_file_repo.root_path), filename=task_filename, content=context.json() + ) + if task_filename in changed_files.docs: + logger.error( + f"Log to expose potential file name conflicts: {coding_doc.json()} & " + f"{changed_files.docs[task_filename].json()}" + ) + changed_files.docs[task_filename] = coding_doc + self.todos = [WriteCode(context=i, llm=self._llm) for i in changed_files.docs.values()] + # 用户直接修改的code + dependency = await CONFIG.git_repo.get_dependency() + for filename in changed_src_files: + if filename in changed_files.docs: continue - self.todos = self.parse_tasks(message) - return 1 - - return 0 + coding_doc = await self._new_coding_doc( + filename=filename, + src_file_repo=src_file_repo, + task_file_repo=task_file_repo, + design_file_repo=design_file_repo, + dependency=dependency, + ) + changed_files.docs[filename] = coding_doc + self.todos.append(WriteCode(context=coding_doc, llm=self._llm)) + # 仅单测 + if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files.docs: + context = await self._new_coding_context( + filename=CONFIG.REQA_FILENAME, + src_file_repo=src_file_repo, + task_file_repo=task_file_repo, + design_file_repo=design_file_repo, + dependency=dependency, + ) + self.publish_message(Message(content=context.json(), instruct_content=context, cause_by=WriteCode)) + + if self.todos: + self._rc.todo = self.todos[0] + return self._rc.todo # For agent store + + @staticmethod + async def _new_coding_context( + filename, src_file_repo, task_file_repo, design_file_repo, dependency + ) -> CodingContext: + old_code_doc = await src_file_repo.get(filename) + if not old_code_doc: + old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="") + dependencies = {Path(i) for i in dependency.get(old_code_doc.root_relative_path)} + task_doc = None + design_doc = None + for i in dependencies: + if str(i.parent) == TASK_FILE_REPO: + task_doc = task_file_repo.get(i.filename) + elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: + design_doc = design_file_repo.get(i.filename) + context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) + return context + + @staticmethod + async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency): + context = await Engineer._new_coding_context( + filename, src_file_repo, task_file_repo, design_file_repo, dependency + ) + coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) + return coding_doc diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 760b65736..b57b64a7e 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -151,13 +151,6 @@ async def _debug_error(self, msg): ) self.publish_message(msg) - async def _observe(self) -> int: - await super()._observe() - self._rc.news = [ - msg for msg in self._rc.news if self.profile in msg.send_to - ] # only relevant msgs count as observed news - return len(self._rc.news) - async def _act(self) -> Message: if self.test_round > self.test_round_allowed: result_msg = Message( diff --git a/metagpt/schema.py b/metagpt/schema.py index 674091e4c..6a707af3e 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -238,3 +238,10 @@ def load(self, v) -> "MessageQueue": logger.warning(f"JSON load failed: {v}, error:{e}") return q + + +class CodingContext(BaseModel): + filename: str + design_doc: Document + task_doc: Document + code_doc: Document diff --git a/startup.py b/startup.py index d5a6bb07b..1a59e7fa2 100644 --- a/startup.py +++ b/startup.py @@ -56,6 +56,7 @@ def main( run_tests: bool = False, implement: bool = True, project_path: str = None, + reqa_file: str = None, ): """ We are a software startup comprised of AI. By investing in us, @@ -71,6 +72,7 @@ def main( :return: """ CONFIG.WORKDIR = project_path + CONFIG.REQA_FILENAME = reqa_file asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) From 66cff9023fe0bd27843b0754110e7d8bda902a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 18:10:44 +0800 Subject: [PATCH 131/232] fixbug: Delete the incorrect message. --- metagpt/roles/engineer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d23d23d55..c0e1b8a10 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -143,7 +143,6 @@ async def _act_mp(self) -> Message: # self.write_file(todo, code) msg = Message(content=code_rsp, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) - self.publish_message(msg) del self.todos[0] logger.info(f"Done {self.get_workspace()} generating.") @@ -160,7 +159,6 @@ async def _act_sp(self) -> Message: file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) - self.publish_message(msg) code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) @@ -202,7 +200,6 @@ async def _act_sp_precision(self) -> Message: file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) - self.publish_message(msg) code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) From 9a3c92ed1192387f28eee11fcca3e08b737f7fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 21:38:39 +0800 Subject: [PATCH 132/232] fixbug: send useless message to nobody from QaEngineer --- metagpt/roles/qa_engineer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index b57b64a7e..23c8d1fdd 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -100,8 +100,8 @@ async def _write_test(self, message: Message) -> None: content=str(file_info), role=self.profile, cause_by=WriteTest, - sent_from=self.profile, - send_to=self.profile, + sent_from=self, + send_to=self, ) self.publish_message(msg) @@ -182,5 +182,6 @@ async def _act(self) -> Message: role=self.profile, cause_by=WriteTest, sent_from=self.profile, + send_to="" ) return result_msg From 3e8bba70bcd20fda71151f3b526171d96818cdf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 21:38:51 +0800 Subject: [PATCH 133/232] fixbug: send useless message to nobody from QaEngineer --- metagpt/roles/qa_engineer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 23c8d1fdd..59a4135b8 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -146,8 +146,8 @@ async def _debug_error(self, msg): content=file_info, role=self.profile, cause_by=DebugError, - sent_from=self.profile, - send_to=recipient, + sent_from=self, + send_to=self, ) self.publish_message(msg) @@ -158,6 +158,7 @@ async def _act(self) -> Message: role=self.profile, cause_by=WriteTest, sent_from=self.profile, + send_to="" ) return result_msg From ec3dd004af6000ec44ec6bb2cd6ed49d39e09ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 22:41:44 +0800 Subject: [PATCH 134/232] feat: Change the operation of transmitting file content during the QA process to transmitting file names instead. --- metagpt/actions/debug_error.py | 12 +-- metagpt/actions/run_code.py | 24 ++--- metagpt/actions/write_test.py | 16 +-- metagpt/const.py | 2 + metagpt/roles/engineer.py | 41 +++----- metagpt/roles/qa_engineer.py | 178 +++++++++++++++----------------- metagpt/schema.py | 26 +++++ metagpt/utils/git_repository.py | 7 +- 8 files changed, 157 insertions(+), 149 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 304b1bc3e..a55f13dad 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : debug_error.py """ -import re from metagpt.actions.action import Action from metagpt.logs import logger @@ -36,18 +35,17 @@ def __init__(self, name="DebugError", context=None, llm=None): # fixed_code = await self._aask(prompt) # return fixed_code - async def run(self, context): - if "PASS" in context: + async def run(self, *args, **kwargs) -> str: + if "PASS" in self.context.output: return "", "the original code works fine, no need to debug" - file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) - + file_name = self.context.code_filename logger.info(f"Debug and rewrite {file_name}") - prompt = PROMPT_TEMPLATE.format(context=context) + prompt = PROMPT_TEMPLATE.format(context=self.context.output) rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=rsp) - return file_name, code + return code diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f69d2cd1a..f2d323f06 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -98,24 +98,22 @@ async def run_script(cls, working_directory, additional_python_paths=[], command stdout, stderr = process.communicate() return stdout.decode("utf-8"), stderr.decode("utf-8") - async def run( - self, code, mode="script", code_file_name="", test_code="", test_file_name="", command=[], **kwargs - ) -> str: - logger.info(f"Running {' '.join(command)}") - if mode == "script": - outs, errs = await self.run_script(command=command, **kwargs) - elif mode == "text": - outs, errs = await self.run_text(code=code) + async def run(self, *args, **kwargs) -> str: + logger.info(f"Running {' '.join(self.context.command)}") + if self.context.mode == "script": + outs, errs = await self.run_script(command=self.context.command, **kwargs) + elif self.context.mode == "text": + outs, errs = await self.run_text(code=self.context.code) logger.info(f"{outs=}") logger.info(f"{errs=}") context = CONTEXT.format( - code=code, - code_file_name=code_file_name, - test_code=test_code, - test_file_name=test_file_name, - command=" ".join(command), + code=self.context.code, + code_file_name=self.context.code_filename, + test_code=self.context.test_code, + test_file_name=self.context.test_filename, + command=" ".join(self.context.command), outs=outs[:500], # outs might be long but they are not important, truncate them to avoid token overflow errs=errs[:10000], # truncate errors to avoid token overflow ) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 35ff36dc2..9a9671bab 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -6,7 +6,9 @@ @File : environment.py """ from metagpt.actions.action import Action +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.schema import TestingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -47,12 +49,12 @@ async def write_code(self, prompt): code = code_rsp return code - async def run(self, code_to_test, test_file_name, source_file_path, workspace): + async def run(self, *args, **kwargs) -> TestingContext: prompt = PROMPT_TEMPLATE.format( - code_to_test=code_to_test, - test_file_name=test_file_name, - source_file_path=source_file_path, - workspace=workspace, + code_to_test=self.context.code_doc.content, + test_file_name=self.context.test_doc.filename, + source_file_path=self.context.code_doc.root_relative_path, + workspace=CONFIG.git_repo.workdir, ) - code = await self.write_code(prompt) - return code + self.context.test_doc.content = await self.write_code(prompt) + return self.context diff --git a/metagpt/const.py b/metagpt/const.py index 7ee06ff7d..e97ffdb7d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -61,3 +61,5 @@ def get_project_root(): SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" +TEST_CODES_FILE_REPO = "tests" +OUTPUTS_FILE_REPO = "outputs" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 8852d55f1..89827a1ca 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -15,6 +15,7 @@ import json from pathlib import Path +from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.config import CONFIG @@ -22,7 +23,6 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import CodingContext, Document, Documents, Message -from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP class Engineer(Role): @@ -60,8 +60,8 @@ def _parse_tasks(task_msg: Document) -> list[str]: m = json.loads(task_msg.content) return m.get("Task list") - async def _act_sp_precision(self, review=False) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + async def _act_sp_precision(self, review=False) -> Set[str]: + changed_files = set() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) for todo in self.todos: """ @@ -88,24 +88,27 @@ async def _act_sp_precision(self, review=False) -> Message: content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode ) self._rc.memory.add(msg) - self.publish_message(msg) - code_msg = coding_context.filename + FILENAME_CODE_SEP + str(coding_context.code_doc.root_relative_path) - code_msg_all.append(code_msg) + changed_files.add(coding_context.code_doc.filename) + return changed_files + + async def _act(self) -> Message: + """Determines the mode of action based on whether code review is used.""" + changed_files = await self._act_sp_precision(review=self.use_code_review) + # 仅单测 + if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: + changed_files.add(CONFIG.REQA_FILENAME) + + from metagpt.roles import QaEngineer # 避免循环引用 - logger.info(f"Done {CONFIG.src_workspace} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), + content="\n".join(changed_files), role=self.profile, - cause_by=self._rc.todo, - send_to="Edward", + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=QaEngineer, ) return msg - async def _act(self) -> Message: - """Determines the mode of action based on whether code review is used.""" - return await self._act_sp_precision(review=self.use_code_review) - async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name @@ -153,16 +156,6 @@ async def _think(self) -> Action | None: ) changed_files.docs[filename] = coding_doc self.todos.append(WriteCode(context=coding_doc, llm=self._llm)) - # 仅单测 - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files.docs: - context = await self._new_coding_context( - filename=CONFIG.REQA_FILENAME, - src_file_repo=src_file_repo, - task_file_repo=task_file_repo, - design_file_repo=design_file_repo, - dependency=dependency, - ) - self.publish_message(Message(content=context.json(), instruct_content=context, cause_by=WriteCode)) if self.todos: self._rc.todo = self.todos[0] diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index b57b64a7e..1520a830a 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -7,23 +7,15 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. """ -import os -from pathlib import Path +import json -from metagpt.actions import ( - DebugError, - RunCode, - WriteCode, - WriteCodeReview, - WriteDesign, - WriteTest, -) -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest +from metagpt.config import CONFIG +from metagpt.const import OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser, any_to_str_set, parse_recipient -from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP +from metagpt.schema import Document, Message, RunCodeContext, TestingContext +from metagpt.utils.common import CodeParser, any_to_str_set class QaEngineer(Role): @@ -49,107 +41,98 @@ def parse_workspace(cls, system_design_msg: Message) -> str: return system_design_msg.instruct_content.dict().get("Python package name") return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return WORKSPACE_ROOT / "src" - workspace = self.parse_workspace(msg) - # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. - if return_proj_dir: - return WORKSPACE_ROOT / workspace - # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() / "tests" - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - async def _write_test(self, message: Message) -> None: - code_msgs = message.content.split(MSG_SEP) - # result_msg_all = [] - for code_msg in code_msgs: + changed_files = message.content.splitlines() + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) + for filename in changed_files: # write tests - file_name, file_path = code_msg.split(FILENAME_CODE_SEP) - code_to_test = open(file_path, "r").read() - if "test" in file_name: - continue # Engineer might write some test files, skip testing a test file - test_file_name = "test_" + file_name - test_file_path = self.get_workspace() / "tests" / test_file_name - logger.info(f"Writing {test_file_name}..") - test_code = await WriteTest().run( - code_to_test=code_to_test, - test_file_name=test_file_name, - # source_file_name=file_name, - source_file_path=file_path, - workspace=self.get_workspace(), + if not filename or "test" in filename: + continue + code_doc = await src_file_repo.get(filename) + test_doc = await tests_file_repo.get("test_" + code_doc.filename) + if not test_doc: + test_doc = Document( + root_path=str(tests_file_repo.root_path), filename="test_" + code_doc.filename, content="" + ) + logger.info(f"Writing {test_doc.filename}..") + context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc) + context = await WriteTest(context=context, llm=self._llm).run() + await tests_file_repo.save( + filename=context.test_doc.filename, + content=context.test_doc.content, + dependencies={context.code_doc.root_relative_path}, ) - self.write_file(test_file_name, test_code) # prepare context for run tests in next round - command = ["python", f"tests/{test_file_name}"] - file_info = { - "file_name": file_name, - "file_path": str(file_path), - "test_file_name": test_file_name, - "test_file_path": str(test_file_path), - "command": command, - } + run_code_context = RunCodeContext( + command=["python", context.test_doc.root_relative_path], + code_filename=context.code_doc.filename, + test_filename=context.test_doc.filename, + working_directory=str(CONFIG.git_repo.workdir), + additional_python_paths=[CONFIG.src_workspace], + ) + msg = Message( - content=str(file_info), + content=run_code_context.json(), role=self.profile, cause_by=WriteTest, - sent_from=self.profile, - send_to=self.profile, + sent_from=self, + send_to=self, ) self.publish_message(msg) - logger.info(f"Done {self.get_workspace()}/tests generating.") + logger.info(f"Done {str(tests_file_repo.workdir)} generating.") async def _run_code(self, msg): - file_info = eval(msg.content) - development_file_path = file_info["file_path"] - test_file_path = file_info["test_file_path"] - if not os.path.exists(development_file_path) or not os.path.exists(test_file_path): + m = json.loads(msg.content) + run_code_context = RunCodeContext(**m) + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + src_doc = await src_file_repo.get(run_code_context.code_filename) + if not src_doc: return - - development_code = open(development_file_path, "r").read() - test_code = open(test_file_path, "r").read() - proj_dir = self.get_workspace() - development_code_dir = self.get_workspace(return_proj_dir=False) - - result_msg = await RunCode().run( - mode="script", - code=development_code, - code_file_name=file_info["file_name"], - test_code=test_code, - test_file_name=file_info["test_file_name"], - command=file_info["command"], - working_directory=proj_dir, # workspace/package_name, will run tests/test_xxx.py here - additional_python_paths=[development_code_dir], # workspace/package_name/package_name, - # import statement inside package code needs this + test_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) + test_doc = await test_file_repo.get(run_code_context.test_filename) + if not test_doc: + return + run_code_context.code = src_doc.content + run_code_context.test_code = test_doc.content + result_msg = await RunCode(context=run_code_context, llm=self._llm).run() + outputs_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) + run_code_context.output_filename = run_code_context.test_filename + ".log" + await outputs_file_repo.save( + filename=run_code_context.output_filename, + content=result_msg, + dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, + ) + run_code_context.code = None + run_code_context.test_code = None + msg = Message( + content=run_code_context.json(), role=self.profile, cause_by=RunCode, sent_from=self, send_to=self ) - - recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself - content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message(content=content, role=self.profile, cause_by=RunCode, sent_from=self.profile, send_to=recipient) self.publish_message(msg) async def _debug_error(self, msg): - file_info, context = msg.content.split(FILENAME_CODE_SEP) - file_name, code = await DebugError().run(context) - if file_name: - self.write_file(file_name, code) - recipient = msg.sent_from # send back to the one who ran the code for another run, might be one's self - msg = Message( - content=file_info, - role=self.profile, - cause_by=DebugError, - sent_from=self.profile, - send_to=recipient, - ) - self.publish_message(msg) + m = json.loads(msg.context) + run_code_context = RunCodeContext(**m) + output_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) + output_doc = await output_file_repo.get(run_code_context.output_filename) + if not output_doc: + return + run_code_context.output = output_doc.content + code = await DebugError(context=run_code_context, llm=self._llm).run() + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + await src_file_repo.save(filename=run_code_context.code_filename, content=code) + run_code_context.output = None + run_code_context.output_filename = None + msg = Message( + content=run_code_context.json(), + role=self.profile, + cause_by=DebugError, + sent_from=self, + send_to=self, + ) + self.publish_message(msg) async def _act(self) -> Message: if self.test_round > self.test_round_allowed: @@ -182,5 +165,6 @@ async def _act(self) -> Message: role=self.profile, cause_by=WriteTest, sent_from=self.profile, + send_to="", ) return result_msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 6a707af3e..5cc7cdb2d 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -19,6 +19,7 @@ from pydantic import BaseModel, Field +from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_CAUSE_BY, MESSAGE_ROUTE_FROM, @@ -59,6 +60,12 @@ def root_relative_path(self): """ return os.path.join(self.root_path, self.filename) + @property + def full_path(self): + if not CONFIG.git_repo: + return None + return str(CONFIG.git_repo.workdir / self.root_path / self.filename) + class Documents(BaseModel): """A class representing a collection of documents. @@ -245,3 +252,22 @@ class CodingContext(BaseModel): design_doc: Document task_doc: Document code_doc: Document + + +class TestingContext(BaseModel): + filename: str + code_doc: Document + test_doc: Document + + +class RunCodeContext(BaseModel): + mode: str = "script" + code: Optional[str] + code_filename: str = "" + test_code: Optional[str] + test_filename: str = "" + command: List[str] = Field(default_factory=list) + working_directory: str = "" + additional_python_paths: List[str] = Field(default_factory=list) + output_filename: Optional[str] + output: Optional[str] diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 2a4fb4a4d..c2eb2360e 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -160,7 +160,12 @@ def new_file_repository(self, relative_path: Path | str = ".") -> FileRepository :param relative_path: The relative path to the file repository within the Git repository. :return: A new instance of FileRepository. """ - return FileRepository(git_repo=self, relative_path=Path(relative_path)) + path = Path(relative_path) + try: + path = path.relative_to(self.workdir) + except ValueError: + path = relative_path + return FileRepository(git_repo=self, relative_path=Path(path)) async def get_dependency(self) -> DependencyFile: """Get the dependency file associated with the Git repository. From 6d77cd89c86c4dd01007933e2f19352ef32d7dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 22:54:22 +0800 Subject: [PATCH 135/232] refactor: delete useless codes --- metagpt/roles/qa_engineer.py | 8 +------- metagpt/roles/role.py | 4 ---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 7a2b7cbd4..6f0738294 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -15,7 +15,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext -from metagpt.utils.common import CodeParser, any_to_str_set +from metagpt.utils.common import any_to_str_set class QaEngineer(Role): @@ -35,12 +35,6 @@ def __init__( self.test_round = 0 self.test_round_allowed = test_round_allowed - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - async def _write_test(self, message: Message) -> None: changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index d1e65a4e0..2e3bcbbd5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -164,10 +164,6 @@ def set_env(self, env: "Environment"): if env: env.set_subscription(self, self._subscription) - def get_env(self): - """Return the environment in which the role works.""" - return self._rc.env - @property def profile(self): """Get the role description (position)""" From 10d9f33150a5e4b96bf904098db7ebaa7dc2aeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 23:04:41 +0800 Subject: [PATCH 136/232] refactor: use MESSAGE_ROUTE_TO_NONE --- metagpt/const.py | 1 + metagpt/roles/qa_engineer.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index e97ffdb7d..311712013 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -49,6 +49,7 @@ def get_project_root(): MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" MESSAGE_ROUTE_TO_ALL = "" +MESSAGE_ROUTE_TO_NONE = "" REQUIREMENT_FILENAME = "requirement.txt" DOCS_FILE_REPO = "docs" diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 6f0738294..eac30413a 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,7 +11,7 @@ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG -from metagpt.const import OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO +from metagpt.const import MESSAGE_ROUTE_TO_NONE, OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext @@ -66,7 +66,6 @@ async def _write_test(self, message: Message) -> None: working_directory=str(CONFIG.git_repo.workdir), additional_python_paths=[CONFIG.src_workspace], ) - msg = Message( content=run_code_context.json(), role=self.profile, @@ -135,7 +134,7 @@ async def _act(self) -> Message: role=self.profile, cause_by=WriteTest, sent_from=self.profile, - send_to="", + send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg @@ -160,6 +159,6 @@ async def _act(self) -> Message: role=self.profile, cause_by=WriteTest, sent_from=self.profile, - send_to="", + send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg From 75dcc8d5341f6eda98ced66a032369953c75445c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 13:30:00 +0800 Subject: [PATCH 137/232] fixbug: DebugError --- metagpt/actions/debug_error.py | 5 ++- metagpt/actions/run_code.py | 25 +++++++++-- metagpt/roles/qa_engineer.py | 77 +++++++++++++++++--------------- metagpt/schema.py | 24 ++++++++++ metagpt/utils/common.py | 8 +++- metagpt/utils/file_repository.py | 11 ++++- 6 files changed, 107 insertions(+), 43 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index a55f13dad..7a12e18f8 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : debug_error.py """ +import re from metagpt.actions.action import Action from metagpt.logs import logger @@ -36,7 +37,9 @@ def __init__(self, name="DebugError", context=None, llm=None): # return fixed_code async def run(self, *args, **kwargs) -> str: - if "PASS" in self.context.output: + pattern = r"Ran (\d+) tests in ([\d.]+)s\n\nOK" + matches = re.search(pattern, self.context.output) + if matches: return "", "the original code works fine, no need to debug" file_name = self.context.code_filename diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f2d323f06..b244577a7 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -51,8 +51,14 @@ ## Running Command {command} ## Running Output -standard output: {outs}; -standard errors: {errs}; +standard output: +```text +{outs} +``` +standard errors: +```text +{errs} +``` """ @@ -84,10 +90,19 @@ async def run_script(cls, working_directory, additional_python_paths=[], command additional_python_paths = ":".join(additional_python_paths) env["PYTHONPATH"] = additional_python_paths + ":" + env.get("PYTHONPATH", "") + install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + logger.info(" ".join(install_command)) + subprocess.run(install_command, check=True, cwd=working_directory, env=env) + + install_pytest_command = ["python", "-m", "pip", "install", "pytest"] + logger.info(" ".join(install_pytest_command)) + subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + # Start the subprocess process = subprocess.Popen( command, cwd=working_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) + logger.info(" ".join(command)) try: # Wait for the process to complete, with a timeout @@ -101,7 +116,11 @@ async def run_script(cls, working_directory, additional_python_paths=[], command async def run(self, *args, **kwargs) -> str: logger.info(f"Running {' '.join(self.context.command)}") if self.context.mode == "script": - outs, errs = await self.run_script(command=self.context.command, **kwargs) + outs, errs = await self.run_script( + command=self.context.command, + working_directory=self.context.working_directory, + additional_python_paths=self.context.additional_python_paths, + ) elif self.context.mode == "text": outs, errs = await self.run_text(code=self.context.code) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index eac30413a..f950efef4 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -7,15 +7,13 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. """ -import json - from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG from metagpt.const import MESSAGE_ROUTE_TO_NONE, OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext -from metagpt.utils.common import any_to_str_set +from metagpt.utils.common import any_to_str_set, parse_recipient class QaEngineer(Role): @@ -64,68 +62,76 @@ async def _write_test(self, message: Message) -> None: code_filename=context.code_doc.filename, test_filename=context.test_doc.filename, working_directory=str(CONFIG.git_repo.workdir), - additional_python_paths=[CONFIG.src_workspace], + additional_python_paths=[str(CONFIG.src_workspace)], ) - msg = Message( - content=run_code_context.json(), - role=self.profile, - cause_by=WriteTest, - sent_from=self, - send_to=self, + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=WriteTest, + sent_from=self, + send_to=self, + ) ) - self.publish_message(msg) logger.info(f"Done {str(tests_file_repo.workdir)} generating.") async def _run_code(self, msg): - m = json.loads(msg.content) - run_code_context = RunCodeContext(**m) - src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - src_doc = await src_file_repo.get(run_code_context.code_filename) + run_code_context = RunCodeContext.loads(msg.content) + src_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(run_code_context.code_filename) if not src_doc: return - test_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) - test_doc = await test_file_repo.get(run_code_context.test_filename) + test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(run_code_context.test_filename) if not test_doc: return run_code_context.code = src_doc.content run_code_context.test_code = test_doc.content result_msg = await RunCode(context=run_code_context, llm=self._llm).run() - outputs_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) - run_code_context.output_filename = run_code_context.test_filename + ".log" - await outputs_file_repo.save( + run_code_context.output_filename = run_code_context.test_filename + ".md" + await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).save( filename=run_code_context.output_filename, content=result_msg, dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, ) run_code_context.code = None run_code_context.test_code = None - msg = Message( - content=run_code_context.json(), role=self.profile, cause_by=RunCode, sent_from=self, send_to=self + recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself + mappings = { + "Engineer": "Alex", + "QaEngineer": "Edward", + } + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=RunCode, + sent_from=self, + send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE), + ) ) - self.publish_message(msg) async def _debug_error(self, msg): - m = json.loads(msg.context) - run_code_context = RunCodeContext(**m) + run_code_context = RunCodeContext.loads(msg.content) output_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) output_doc = await output_file_repo.get(run_code_context.output_filename) if not output_doc: return run_code_context.output = output_doc.content code = await DebugError(context=run_code_context, llm=self._llm).run() - src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - await src_file_repo.save(filename=run_code_context.code_filename, content=code) + await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).save( + filename=run_code_context.code_filename, content=code + ) run_code_context.output = None run_code_context.output_filename = None - msg = Message( - content=run_code_context.json(), - role=self.profile, - cause_by=DebugError, - sent_from=self, - send_to=self, + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=DebugError, + sent_from=self, + send_to=self, + ) ) - self.publish_message(msg) async def _act(self) -> Message: if self.test_round > self.test_round_allowed: @@ -154,11 +160,10 @@ async def _act(self) -> Message: # I ran my test code, time to fix bugs, if any await self._debug_error(msg) self.test_round += 1 - result_msg = Message( + return Message( content=f"Round {self.test_round} of tests done", role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) - return result_msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 5cc7cdb2d..53a22f0e6 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -253,12 +253,28 @@ class CodingContext(BaseModel): task_doc: Document code_doc: Document + @staticmethod + def loads(val: str) -> CodingContext | None: + try: + m = json.loads(val) + return CodingContext(**m) + except Exception: + return None + class TestingContext(BaseModel): filename: str code_doc: Document test_doc: Document + @staticmethod + def loads(val: str) -> TestingContext | None: + try: + m = json.loads(val) + return TestingContext(**m) + except Exception: + return None + class RunCodeContext(BaseModel): mode: str = "script" @@ -271,3 +287,11 @@ class RunCodeContext(BaseModel): additional_python_paths: List[str] = Field(default_factory=list) output_filename: Optional[str] output: Optional[str] + + @staticmethod + def loads(val: str) -> RunCodeContext | None: + try: + m = json.loads(val) + return RunCodeContext(**m) + except Exception: + return None diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 798acf214..9002a8dfb 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -304,7 +304,13 @@ def print_members(module, indent=0): def parse_recipient(text): pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now recipient = re.search(pattern, text) - return recipient.group(1) if recipient else "" + if recipient: + return recipient.group(1) + pattern = r"Send To:\s*([A-Za-z]+)\s*?" + recipient = re.search(pattern, text) + if recipient: + return recipient.group(1) + return "" def get_class_name(cls) -> str: diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 62ba99d42..8de4bdf5b 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -96,8 +96,15 @@ async def get(self, filename: Path | str) -> Document | None: path_name = self.workdir / filename if not path_name.exists(): return None - async with aiofiles.open(str(path_name), mode="r") as reader: - doc.content = await reader.read() + try: + async with aiofiles.open(str(path_name), mode="r") as reader: + doc.content = await reader.read() + except FileNotFoundError as e: + logger.info(f"open {str(path_name)} failed:{e}") + return None + except Exception as e: + logger.info(f"open {str(path_name)} failed:{e}") + return None return doc async def get_all(self) -> List[Document]: From 45be71d9bf2c8cc6619c3c3062d2b37022cebe92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 13:36:35 +0800 Subject: [PATCH 138/232] fixbug: DebugError --- metagpt/roles/qa_engineer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index f950efef4..68138d925 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -112,8 +112,7 @@ async def _run_code(self, msg): async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) - output_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) - output_doc = await output_file_repo.get(run_code_context.output_filename) + output_doc = await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).get(run_code_context.output_filename) if not output_doc: return run_code_context.output = output_doc.content From 938fa8a446de3d1fbb50efc780577a1854ec6c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 13:48:25 +0800 Subject: [PATCH 139/232] feat: git archive --- metagpt/software_company.py | 2 ++ metagpt/utils/git_repository.py | 1 + 2 files changed, 3 insertions(+) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index d3c2c463b..5aa0864e0 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -59,4 +59,6 @@ async def run(self, n_round=3): logger.debug(f"{n_round=}") self._check_balance() await self.environment.run() + if CONFIG.git_repo: + CONFIG.git_repo.archive() return self.environment.history diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index c2eb2360e..660561bf3 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -151,6 +151,7 @@ def archive(self, comments="Archive"): :param comments: Comments for the archive commit. """ + logger.info(f"Archive: {[list(self.changed_files.keys())]}") self.add_change(self.changed_files) self.commit(comments) From 8ce6914df21b0799db04a968f3243a591ff14c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 14:04:01 +0800 Subject: [PATCH 140/232] feat: git archive --- metagpt/utils/git_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 660561bf3..ace0cf8a2 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -151,7 +151,7 @@ def archive(self, comments="Archive"): :param comments: Comments for the archive commit. """ - logger.info(f"Archive: {[list(self.changed_files.keys())]}") + logger.info(f"Archive: {list(self.changed_files.keys())}") self.add_change(self.changed_files) self.commit(comments) From 882f22da352f8099af6fc0974a292c4866cb6c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 19:56:27 +0800 Subject: [PATCH 141/232] =?UTF-8?q?feat:=20=E6=B5=81=E7=A8=8B=E8=B0=83?= =?UTF-8?q?=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/debug_error.py | 45 +++++++++---- metagpt/actions/design_api.py | 93 +++++++++++++-------------- metagpt/actions/project_management.py | 41 +++++++++++- metagpt/actions/run_code.py | 29 +++++---- metagpt/actions/write_code.py | 69 +++++++++++--------- metagpt/actions/write_prd.py | 74 +++++++++++++++++++-- metagpt/const.py | 2 +- metagpt/roles/engineer.py | 8 ++- metagpt/roles/qa_engineer.py | 26 ++++---- metagpt/schema.py | 14 ++++ metagpt/utils/git_repository.py | 9 ++- 11 files changed, 274 insertions(+), 136 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 7a12e18f8..d0c3652b4 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -8,7 +8,10 @@ import re from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger +from metagpt.schema import RunCodeResult from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -19,7 +22,20 @@ then rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. The message is as follows: -{context} +# Legacy Code +```python +{code} +``` +--- +# Unit Test Code +```python +{test_code} +``` +--- +# Console logs +```text +{logs} +``` --- Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. @@ -30,25 +46,26 @@ class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) - # async def run(self, code, error): - # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - # f"\n\n{error}\n\nPlease try to fix the error in this code." - # fixed_code = await self._aask(prompt) - # return fixed_code - async def run(self, *args, **kwargs) -> str: + output_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get(self.context.output_filename) + if not output_doc: + return "" + output_detail = RunCodeResult.loads(output_doc.content) pattern = r"Ran (\d+) tests in ([\d.]+)s\n\nOK" - matches = re.search(pattern, self.context.output) + matches = re.search(pattern, output_detail.stderr) if matches: - return "", "the original code works fine, no need to debug" + return "" - file_name = self.context.code_filename - logger.info(f"Debug and rewrite {file_name}") - - prompt = PROMPT_TEMPLATE.format(context=self.context.output) + logger.info(f"Debug and rewrite {self.context.code_filename}") + code_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(self.context.code_filename) + if not code_doc: + return "" + test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(self.context.test_filename) + if not test_doc: + return "" + prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=rsp) return code diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index a8f89473d..02f87bc47 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -6,7 +6,6 @@ @File : design_api.py """ import json -import shutil from pathlib import Path from typing import List @@ -18,13 +17,11 @@ SEQ_FLOW_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, SYSTEM_DESIGN_PDF_FILE_REPO, - WORKSPACE_ROOT, ) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -157,6 +154,34 @@ class Game{ "Anything UNCLEAR": (str, ...), } +MERGE_PROMPT = """ +## Old Design +{old_design} + +## Context +{context} + +----- +Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a state-of-the-art (SOTA) Python system compliant with PEP8. Additionally, the objective is to optimize the use of high-quality open-source tools. +Requirement: Fill in the following missing information based on the context, each section name is a key in json +Max Output: 8192 chars or 2048 tokens. Try to use them up. + +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. + +## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores + +## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here + +## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. + +## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, +and only output the json inside this tag, nothing else +""" + class WriteDesign(Action): def __init__(self, name, context=None, llm=None): @@ -167,50 +192,6 @@ def __init__(self, name, context=None, llm=None): "clearly and in detail." ) - def recreate_workspace(self, workspace: Path): - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # Folder does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) - - async def _save_prd(self, docs_path, resources_path, context): - prd_file = docs_path / "prd.md" - if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: - quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") - - if context[-1].instruct_content: - logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) - - async def _save_system_design(self, docs_path, resources_path, system_design): - data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) - seq_flow = system_design.instruct_content.dict()[ - "Program call flow" - ] # CodeParser.parse_code(block="Program call flow", text=content) - await mermaid_to_file(data_api_design, resources_path / "data_api_design") - await mermaid_to_file(seq_flow, resources_path / "seq_flow") - system_design_file = docs_path / "system_design.md" - logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) - - async def _save(self, context, system_design): - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["Python package name"] - else: - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name - self.recreate_workspace(workspace) - docs_path = workspace / "docs" - resources_path = workspace / "resources" - docs_path.mkdir(parents=True, exist_ok=True) - resources_path.mkdir(parents=True, exist_ok=True) - await self._save_prd(docs_path, resources_path, context) - await self._save_system_design(docs_path, resources_path, system_design) - async def run(self, with_messages, format=CONFIG.prompt_format): # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) @@ -234,7 +215,8 @@ async def run(self, with_messages, format=CONFIG.prompt_format): filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) changed_files.docs[filename] = doc - + if not changed_files.docs: + logger.info("Nothing has changed.") # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=changed_files.json(), instruct_content=changed_files) @@ -253,10 +235,21 @@ async def _new_system_design(self, context, format=CONFIG.prompt_format): await self._rename_workspace(system_design) return system_design - async def _merge(self, prd_doc, system_design_doc): + async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): + prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) + # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python + # package name" contain space, have to use setattr + setattr( + system_design.instruct_content, + "Python package name", + system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), + ) + system_design_doc.content = system_design.instruct_content.json() return system_design_doc - async def _rename_workspace(self, system_design): + @staticmethod + async def _rename_workspace(system_design): if CONFIG.WORKDIR: # 已经指定了在旧版本上更新 return diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 686aa3689..4fd944027 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -17,6 +17,7 @@ TASK_PDF_FILE_REPO, WORKSPACE_ROOT, ) +from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -169,6 +170,35 @@ "Anything UNCLEAR": (str, ...), } +MERGE_PROMPT = """ +# Context +{context} + +## Old Tasks +{old_tasks} +----- +Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. +Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. + +## Required Python third-party packages: Provided in requirements.txt format + +## Required Other language third-party packages: Provided in requirements.txt format + +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + +## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first + +## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first + +## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +and only output the json inside this tag, nothing else +""" + class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): @@ -209,6 +239,8 @@ async def run(self, with_messages, format=CONFIG.prompt_format): ) change_files.docs[filename] = task_doc + if not change_files.docs: + logger.info("Nothing has changed.") # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) @@ -216,7 +248,7 @@ async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo system_design_doc = await system_design_file_repo.get(filename) task_doc = await tasks_file_repo.get(filename) if task_doc: - task_doc = await self._merge(system_design_doc=system_design_doc, task_dock=task_doc) + task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc) else: rsp = await self._run_new_tasks(context=system_design_doc.content) task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) @@ -234,8 +266,11 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format): # self._save(context, rsp) return rsp - async def _merge(self, system_design_doc, task_dock) -> Document: - return task_dock + async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) + task_doc.content = rsp.instruct_content.json() + return task_doc @staticmethod async def _update_requirements(doc): diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index b244577a7..242eaa25d 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -12,6 +12,7 @@ from metagpt.actions.action import Action from metagpt.logs import logger +from metagpt.schema import RunCodeResult PROMPT_TEMPLATE = """ Role: You are a senior development and qa engineer, your role is summarize the code running result. @@ -89,14 +90,7 @@ async def run_script(cls, working_directory, additional_python_paths=[], command additional_python_paths = [working_directory] + additional_python_paths additional_python_paths = ":".join(additional_python_paths) env["PYTHONPATH"] = additional_python_paths + ":" + env.get("PYTHONPATH", "") - - install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] - logger.info(" ".join(install_command)) - subprocess.run(install_command, check=True, cwd=working_directory, env=env) - - install_pytest_command = ["python", "-m", "pip", "install", "pytest"] - logger.info(" ".join(install_pytest_command)) - subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + RunCode._install_dependencies(working_directory=working_directory, env=env) # Start the subprocess process = subprocess.Popen( @@ -113,7 +107,7 @@ async def run_script(cls, working_directory, additional_python_paths=[], command stdout, stderr = process.communicate() return stdout.decode("utf-8"), stderr.decode("utf-8") - async def run(self, *args, **kwargs) -> str: + async def run(self, *args, **kwargs) -> RunCodeResult: logger.info(f"Running {' '.join(self.context.command)}") if self.context.mode == "script": outs, errs = await self.run_script( @@ -139,7 +133,20 @@ async def run(self, *args, **kwargs) -> str: prompt = PROMPT_TEMPLATE.format(context=context) rsp = await self._aask(prompt) + return RunCodeResult(summary=rsp, stdout=outs, stderr=errs) - result = context + rsp + @staticmethod + def _install_dependencies(working_directory, env): + install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + logger.info(" ".join(install_command)) + try: + subprocess.run(install_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") - return result + install_pytest_command = ["python", "-m", "pip", "install", "pytest"] + logger.info(" ".join(install_pytest_command)) + try: + subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index d4d33fe0c..c9b6c3b9e 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,16 +7,15 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. """ -import json from tenacity import retry, stop_after_attempt, wait_fixed -from metagpt.actions import WriteDesign from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.const import TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger -from metagpt.schema import CodingContext -from metagpt.utils.common import CodeParser, any_to_str +from metagpt.schema import CodingContext, RunCodeResult +from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ NOTICE @@ -33,8 +32,25 @@ 7. Do not use public member functions that do not exist in your design. ----- -# Context -{context} +# Design +```json +{design} +``` +----- +# Tasks +```json +{tasks} +``` +----- +# Legacy Code +```python +{code} +``` +----- +# Debug logs +```text +{logs} +``` ----- ## Format example ----- @@ -51,26 +67,6 @@ class WriteCode(Action): def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) - def _is_invalid(self, filename): - return any(i in filename for i in ["mp3", "wav"]) - - def _save(self, context, filename, code): - # logger.info(filename) - # logger.info(code_rsp) - if self._is_invalid(filename): - return - - design = [i for i in context if i.cause_by == any_to_str(WriteDesign)][0] - - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name - if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - ws_path = ws_path / ws_name - code_path = ws_path / filename - code_path.parent.mkdir(parents=True, exist_ok=True) - code_path.write_text(code) - logger.info(f"Saving Code to {code_path}") - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) @@ -78,12 +74,21 @@ async def write_code(self, prompt) -> str: return code async def run(self, *args, **kwargs) -> CodingContext: - m = json.loads(self.context.content) - coding_context = CodingContext(**m) - context = "\n".join( - [coding_context.design_doc.content, coding_context.task_doc.content, coding_context.code_doc.content] + coding_context = CodingContext.loads(self.context.content) + test_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get( + "test_" + coding_context.filename + ".json" + ) + logs = "" + if test_doc: + test_detail = RunCodeResult.loads(test_doc.content) + logs = test_detail.stderr + prompt = PROMPT_TEMPLATE.format( + design=coding_context.design_doc.content, + tasks=coding_context.task_doc.content, + code=coding_context.code_doc.content, + logs=logs, + filename=self.context.filename, ) - prompt = PROMPT_TEMPLATE.format(context=context, filename=self.context.filename) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) coding_context.code_doc.content = code diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 8b03ac29a..532f5bc34 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -219,6 +219,7 @@ }, } + OUTPUT_MAPPING = { "Original Requirements": (str, ...), "Product Goals": (List[str], ...), @@ -231,13 +232,60 @@ "Anything UNCLEAR": (str, ...), } +IS_RELATIVE_PROMPT = """ +## PRD: +{old_prd} + +## New Requirement: +{requirements} + +___ +You are a professional product manager; You need to assess whether the new requirements are relevant to the existing PRD to determine whether to merge the new requirements into this PRD. +Is the newly added requirement in "New Requirement" related to the PRD? +Respond with `YES` if it is related, `NO` if it is not, and provide the reasons. Return the response in JSON format. +""" + +MERGE_PROMPT = """ +# Context +## Original Requirements +{requirements} + + +## Old PRD +{old_prd} +----- +Role: You are a professional product manager; The goal is to merge the newly added requirements into the existing PRD in order to design a concise, usable, and efficient product. +Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design + +## Original Requirements: Provide as Plain text, place the polished complete original requirements here + +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple + +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less + +## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible + +## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + +## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. + +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower + +## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old PRD" format, +and only output the json inside this tag, nothing else +""" + class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: - # 判断哪些需求文档需要重写：调LLM判断新增需求与prd是否相关，若相关就rewrite prd + # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are + # related to the PRD. If they are related, rewrite the PRD. docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) @@ -250,14 +298,16 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc - # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd + # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc - # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 + # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the + # 'publish' message to transition the workflow to the next stage. This design allows room for global + # optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: @@ -278,11 +328,23 @@ async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd - async def _is_relative_to(self, doc1, doc2) -> bool: + async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: + m = json.loads(old_prd_doc.content) + if m.get("Original Requirements") == new_requirement_doc.content: + # There have been no changes in the requirements, so they are considered unrelated. + return False + prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) + res = await self._aask(prompt=prompt) + logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + if "YES" in res: + return True return False - async def _merge(self, doc1, doc2) -> Document: - pass + async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: + prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + prd_doc.content = prd.instruct_content.json() + return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: diff --git a/metagpt/const.py b/metagpt/const.py index 311712013..49965b622 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -63,4 +63,4 @@ def get_project_root(): PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" -OUTPUTS_FILE_REPO = "outputs" +TEST_OUTPUTS_FILE_REPO = "test_outputs" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 89827a1ca..b6ecc4767 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -90,6 +90,8 @@ async def _act_sp_precision(self, review=False) -> Set[str]: self._rc.memory.add(msg) changed_files.add(coding_context.code_doc.filename) + if not changed_files: + logger.info("Nothing has changed.") return changed_files async def _act(self) -> Message: @@ -136,8 +138,8 @@ async def _think(self) -> Action | None: root_path=str(src_file_repo.root_path), filename=task_filename, content=context.json() ) if task_filename in changed_files.docs: - logger.error( - f"Log to expose potential file name conflicts: {coding_doc.json()} & " + logger.warning( + f"Log to expose potential conflicts: {coding_doc.json()} & " f"{changed_files.docs[task_filename].json()}" ) changed_files.docs[task_filename] = coding_doc @@ -168,7 +170,7 @@ async def _new_coding_context( old_code_doc = await src_file_repo.get(filename) if not old_code_doc: old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="") - dependencies = {Path(i) for i in dependency.get(old_code_doc.root_relative_path)} + dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)} task_doc = None design_doc = None for i in dependencies: diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 68138d925..a88b01e37 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -9,7 +9,11 @@ """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO +from metagpt.const import ( + MESSAGE_ROUTE_TO_NONE, + TEST_CODES_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, +) from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext @@ -86,20 +90,17 @@ async def _run_code(self, msg): return run_code_context.code = src_doc.content run_code_context.test_code = test_doc.content - result_msg = await RunCode(context=run_code_context, llm=self._llm).run() - run_code_context.output_filename = run_code_context.test_filename + ".md" - await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).save( + result = await RunCode(context=run_code_context, llm=self._llm).run() + run_code_context.output_filename = run_code_context.test_filename + ".json" + await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).save( filename=run_code_context.output_filename, - content=result_msg, + content=result.json(), dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, ) run_code_context.code = None run_code_context.test_code = None - recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself - mappings = { - "Engineer": "Alex", - "QaEngineer": "Edward", - } + recipient = parse_recipient(result.summary) # the recipient might be Engineer or myself + mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} self.publish_message( Message( content=run_code_context.json(), @@ -112,16 +113,11 @@ async def _run_code(self, msg): async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) - output_doc = await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).get(run_code_context.output_filename) - if not output_doc: - return - run_code_context.output = output_doc.content code = await DebugError(context=run_code_context, llm=self._llm).run() await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).save( filename=run_code_context.code_filename, content=code ) run_code_context.output = None - run_code_context.output_filename = None self.publish_message( Message( content=run_code_context.json(), diff --git a/metagpt/schema.py b/metagpt/schema.py index 53a22f0e6..e910fc866 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -295,3 +295,17 @@ def loads(val: str) -> RunCodeContext | None: return RunCodeContext(**m) except Exception: return None + + +class RunCodeResult(BaseModel): + summary: str + stdout: str + stderr: str + + @staticmethod + def loads(val: str) -> RunCodeResult | None: + try: + m = json.loads(val) + return RunCodeResult(**m) + except Exception: + return None diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index ace0cf8a2..b8e35199b 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -72,7 +72,14 @@ def _init(self, local_path: Path): :param local_path: The local path where the new Git repository will be initialized. """ - self._repository = Repo.init(path=local_path) + self._repository = Repo.init(path=Path(local_path)) + + gitignore_filename = Path(local_path) / ".gitignore" + ignores = ["__pycache__", "*.pyc"] + with open(str(gitignore_filename), mode="w") as writer: + writer.write("\n".join(ignores)) + self._repository.index.add([".gitignore"]) + self._repository.index.commit("Add .gitignore") def add_change(self, files: Dict): """Add or remove files from the staging area based on the provided changes. From e1cabcad492d48804376a238c13747619396f1cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:21:52 +0800 Subject: [PATCH 142/232] feat: +annotation --- metagpt/actions/debug_error.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index d0c3652b4..7fdc2ef5b 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py +@Modified By: mashenquan, 2023/11/27. Divide the context into three components: legacy code, unit test code, and + console log. """ import re From 86c5e5e8e662556204bdc69adf1a050e94962320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:28:07 +0800 Subject: [PATCH 143/232] feat: +annotation --- metagpt/actions/debug_error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 7fdc2ef5b..971f76ca7 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : debug_error.py @Modified By: mashenquan, 2023/11/27. Divide the context into three components: legacy code, unit test code, and - console log. + console log. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. """ import re From 22c5077747b22a4ad7b3bfbe1cd25d867e8c84fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:32:40 +0800 Subject: [PATCH 144/232] feat: +annotation --- metagpt/actions/design_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 02f87bc47..8644aa6a4 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,6 +4,9 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py +@Modified By: mashenquan, 2023/11/27. According to Section 2.2.3.1 of RFC 135, replace file data in the message with + the file name. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration + functionality. """ import json from pathlib import Path From 57d826a40cd0d7bb7a17f522fd6c3099f57bc20d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:44:46 +0800 Subject: [PATCH 145/232] feat: +annotation --- metagpt/actions/design_api.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 8644aa6a4..2f8a306d5 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -196,14 +196,15 @@ def __init__(self, name, context=None, llm=None): ) async def run(self, with_messages, format=CONFIG.prompt_format): - # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 + # Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory. prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) changed_prds = prds_file_repo.changed_files - # 通过git diff来识别docs/system_designs下那些设计文档发生了变动； + # Use `git diff` to identify which design documents in the `docs/system_designs` directory have undergone + # changes. system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files - # 对于那些发生变动的PRD和设计文档，重新生成设计内容； + # For those PRDs and design documents that have undergone changes, regenerate the design content. changed_files = Documents() for filename in changed_prds.keys(): doc = await self._update_system_design( @@ -220,7 +221,8 @@ async def run(self, with_messages, format=CONFIG.prompt_format): changed_files.docs[filename] = doc if not changed_files.docs: logger.info("Nothing has changed.") - # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + # Wait until all files under `docs/system_designs/` are processed before sending the publish message, + # leaving room for global optimization in subsequent steps. return ActionOutput(content=changed_files.json(), instruct_content=changed_files) async def _new_system_design(self, context, format=CONFIG.prompt_format): @@ -253,7 +255,7 @@ async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): @staticmethod async def _rename_workspace(system_design): - if CONFIG.WORKDIR: # 已经指定了在旧版本上更新 + if CONFIG.WORKDIR: # Updating on the old version has already been specified if it's valid. return if isinstance(system_design, ActionOutput): From 759c8378e42c4b8a76d3d96946325ccd4c5f61d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:54:09 +0800 Subject: [PATCH 146/232] feat: +annotation --- metagpt/actions/debug_error.py | 5 +++-- metagpt/actions/design_api.py | 6 +++--- metagpt/actions/project_management.py | 26 +++++--------------------- 3 files changed, 11 insertions(+), 26 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 971f76ca7..e4a15d38d 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,8 +4,9 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py -@Modified By: mashenquan, 2023/11/27. Divide the context into three components: legacy code, unit test code, and - console log. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. +@Modified By: mashenquan, 2023/11/27. + 1. Divide the context into three components: legacy code, unit test code, and console log. + 2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. """ import re diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 2f8a306d5..021edfe72 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,9 +4,9 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py -@Modified By: mashenquan, 2023/11/27. According to Section 2.2.3.1 of RFC 135, replace file data in the message with - the file name. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration - functionality. +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. """ import json from pathlib import Path diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 4fd944027..042f1f01c 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,6 +4,10 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py +@Modified By: mashenquan, 2023/11/27. + 1. Divide the context into three components: legacy code, unit test code, and console log. + 2. Move the document storage operations related to WriteDesign to the save operation of WriteDesign. + 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ import json from typing import List @@ -11,17 +15,10 @@ from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import ( - SYSTEM_DESIGN_FILE_REPO, - TASK_FILE_REPO, - TASK_PDF_FILE_REPO, - WORKSPACE_ROOT, -) +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO from metagpt.logs import logger from metagpt.schema import Document, Documents -from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown templates = { "json": { @@ -204,18 +201,6 @@ class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) - def _save(self, context, rsp): - if context[-1].instruct_content: - ws_name = context[-1].instruct_content.dict()["Python package name"] - else: - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" - file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) - - # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" - requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) - async def run(self, with_messages, format=CONFIG.prompt_format): system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files @@ -263,7 +248,6 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - # self._save(context, rsp) return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: From c483d0d7a3c74f8e66ed41d106013a17e80d7d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 12:00:50 +0800 Subject: [PATCH 147/232] feat: +annotation --- metagpt/actions/project_management.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 042f1f01c..0081fd223 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -208,14 +208,15 @@ async def run(self, with_messages, format=CONFIG.prompt_format): tasks_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) changed_tasks = tasks_file_repo.changed_files change_files = Documents() - # 根据docs/system_designs/下的git head diff识别哪些task文档需要重写 + # Rewrite the system designs that have undergone changes based on the git head diff under + # `docs/system_designs/`. for filename in changed_system_designs: task_doc = await self._update_tasks( filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) change_files.docs[filename] = task_doc - # 根据docs/tasks/下的git head diff识别哪些task文件被用户修改了，需要重写 + # Rewrite the task files that have undergone changes based on the git head diff under docs/tasks/. for filename in changed_tasks: if filename in change_files.docs: continue @@ -226,7 +227,8 @@ async def run(self, with_messages, format=CONFIG.prompt_format): if not change_files.docs: logger.info("Nothing has changed.") - # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + # Wait until all files under `docs/tasks/` are processed before sending the publish message, leaving room for + # global optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo): From 5ea488d37a2c43df63f48a874b4f83e5bd50e832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 12:03:21 +0800 Subject: [PATCH 148/232] feat: +annotation --- metagpt/actions/project_management.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 0081fd223..ee1632612 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -216,7 +216,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format): ) change_files.docs[filename] = task_doc - # Rewrite the task files that have undergone changes based on the git head diff under docs/tasks/. + # Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`. for filename in changed_tasks: if filename in change_files.docs: continue @@ -227,7 +227,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format): if not change_files.docs: logger.info("Nothing has changed.") - # Wait until all files under `docs/tasks/` are processed before sending the publish message, leaving room for + # Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for # global optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) From a405b4759b64648fe8f59c4ed411955c77db5714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 13:49:15 +0800 Subject: [PATCH 149/232] feat: +annotation --- metagpt/actions/run_code.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 242eaa25d..1e7010e52 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -4,6 +4,14 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : run_code.py +@Modified By: mashenquan, 2023/11/27. + 1. Mark the location of Console logs in the PROMPT_TEMPLATE with markdown code-block formatting to enhance + the understanding for the LLM. + 2. Fix bug: Add the "install dependency" operation. + 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into + RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. + 4. According to section 2.2.3.5.7 of RFC 135, change the method of transferring file content + (code files, unit test files, log files) from using the message to using the file name. """ import os import subprocess From 0f03645a8920b50b3dcc67817565cdd73f3e0f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:27:56 +0800 Subject: [PATCH 150/232] feat: +annotation --- metagpt/actions/write_code.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c9b6c3b9e..e9d41bb20 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -6,6 +6,13 @@ @File : write_code.py @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. +@Modified By: mashenquan, 2023-11-27. + 1. Mark the location of Design, Tasks, Legacy Code and Debug logs in the PROMPT_TEMPLATE with markdown + code-block formatting to enhance the understanding for the LLM. + 2. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather + than passing them in when calling the run function. + 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into + RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ from tenacity import retry, stop_after_attempt, wait_fixed From 9c13958f6c498e9d24fcd951e1c6ced84d35bde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:31:01 +0800 Subject: [PATCH 151/232] feat: +annotation --- metagpt/actions/write_code_review.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 10e4aec3b..dae1c965f 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code_review.py +@Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the + WriteCode object, rather than passing them in when calling the run function. """ from tenacity import retry, stop_after_attempt, wait_fixed From 16226a2e11621b91d05feb4a74e97259f95d66b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:35:22 +0800 Subject: [PATCH 152/232] feat: +annotation --- metagpt/actions/write_prd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 532f5bc34..68e0e75ba 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -4,6 +4,9 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd.py +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. """ from __future__ import annotations From 512e205cd0945be9c6d8c6a980b309b286788557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:38:46 +0800 Subject: [PATCH 153/232] feat: +annotation --- metagpt/actions/project_management.py | 2 +- metagpt/actions/write_prd.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index ee1632612..641d21533 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -6,7 +6,7 @@ @File : project_management.py @Modified By: mashenquan, 2023/11/27. 1. Divide the context into three components: legacy code, unit test code, and console log. - 2. Move the document storage operations related to WriteDesign to the save operation of WriteDesign. + 2. Move the document storage operations related to WritePRD from the save operation of WriteDesign. 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ import json diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 68e0e75ba..cc21058b4 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. + 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. """ from __future__ import annotations From fbd24635df779764d9cd5608354ab4b649495f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:40:36 +0800 Subject: [PATCH 154/232] feat: +annotation --- metagpt/actions/write_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 9a9671bab..e980e0831 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 22:12 @Author : alexanderwu @File : environment.py +@Modified By: mashenquan, 2023-11-27. Following the think-act principle, solidify the task parameters when creating the + WriteTest object, rather than passing them in when calling the run function. """ from metagpt.actions.action import Action from metagpt.config import CONFIG From 628ecc0fb7585b749da7d49eb28171395af6b042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:46:25 +0800 Subject: [PATCH 155/232] feat: +annotation --- metagpt/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/config.py b/metagpt/config.py index d059a6a29..a20f58ec1 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -2,6 +2,9 @@ # -*- coding: utf-8 -*- """ Provide configuration, singleton +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.11 of RFC 135, add git repository support. + 2. Add the parameter `src_workspace` for the old version project path. """ import os From 331d74059f18b5fdbb4aedbc8c5ce6a234f7ab4e Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 20 Nov 2023 11:24:46 +0800 Subject: [PATCH 156/232] =?UTF-8?q?1.=20=E5=8A=A8=E4=BD=9C=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20=20=201.=20SummarizeCode=E5=8A=A8=E4=BD=9C=EF=BC=9A?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E5=9F=BA=E4=BA=8E=E4=BB=A3=E7=A0=81=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=80=BB=E7=BB=93=EF=BC=8C=E6=80=9D=E8=80=83bug?= =?UTF-8?q?=E3=80=81=E9=80=BB=E8=BE=91=E3=80=81todo=20=20=202.=20CodeRevie?= =?UTF-8?q?w=E5=8A=A8=E4=BD=9C=E4=BC=98=E5=8C=96=EF=BC=9A=E7=9B=AE?= =?UTF-8?q?=E5=89=8D=E5=BC=BA=E5=88=B6=E8=A6=81=E6=B1=82=E5=9B=9E=E7=AD=94?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E6=9C=89=E6=9B=B4=E9=AB=98=E7=9A=84?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E7=8E=87=E4=BA=86=202.=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84=20=20=201.=20Document=E7=9A=84=E6=A0=87?= =?UTF-8?q?=E5=87=86=E5=8C=96=EF=BC=9AEnv->Repo->Document=EF=BC=8C?= =?UTF-8?q?=E5=85=B6=E4=B8=ADDocument/Asset/Code=E9=83=BD=E5=8F=AA?= =?UTF-8?q?=E7=94=A8Document=20=20=20=20=201.=20=E5=8E=9F=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E6=A3=80=E7=B4=A2=E7=9A=84Document=E6=94=B9=E4=B8=BAI?= =?UTF-8?q?ndexableDocument=20=20=202.=20Repo=E7=BB=93=E6=9E=84=E5=BC=95?= =?UTF-8?q?=E5=85=A5=EF=BC=9A=E7=94=A8=E4=BA=8EDocument=E8=A3=85=E8=BD=BD?= =?UTF-8?q?=E4=B8=8E=E5=85=83=E6=95=B0=E6=8D=AE=E8=A3=85=E8=BD=BD=20=20=20?= =?UTF-8?q?3.=20RepoParser=E5=BC=95=E5=85=A5=EF=BC=9A=E5=86=99=E4=BA=86?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E7=AE=80=E5=8D=95=E7=9A=84AST=20parser?= =?UTF-8?q?=EF=BC=88=E5=90=8E=E7=BB=AD=E5=8F=AF=E8=83=BD=E8=A6=81=E6=8D=A2?= =?UTF-8?q?tree-sitter=EF=BC=89=EF=BC=8C=E7=BB=99=E5=87=BA=E4=BA=86?= =?UTF-8?q?=E6=95=B4=E5=BA=93symbol=203.=20=E9=85=8D=E7=BD=AE=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20=20=201.=20=E9=BB=98=E8=AE=A4=E6=9B=B4=E6=8D=A2?= =?UTF-8?q?=E4=B8=BAgpt-4-1106-preview=EF=BC=8C=E4=BB=A5=E8=8E=B7=E5=BE=97?= =?UTF-8?q?=E6=9C=80=E5=A5=BD=E7=9A=84=E6=95=88=E6=9E=9C=E4=B8=8E=E6=88=90?= =?UTF-8?q?=E6=9C=AC=20=20=202.=20=E6=8F=90=E4=BE=9B~/.metagpt=E4=BD=9C?= =?UTF-8?q?=E4=B8=BA=E9=85=8D=E7=BD=AE=E6=9C=80=E9=AB=98=E4=BC=98=E5=85=88?= =?UTF-8?q?=E7=BA=A7=E7=9B=AE=E5=BD=95=EF=BC=8C=E4=BB=8E=E4=B8=AD=E8=AF=BB?= =?UTF-8?q?=E5=8F=96config.yaml=20=20=203.=20workspace=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E7=81=B5=E6=B4=BB=E6=8C=87=E5=AE=9A=E4=BA=86=EF=BC=8C=E5=9C=A8?= =?UTF-8?q?config=E4=B8=AD=E9=85=8D=E7=BD=AE=204.=20metagpt=E4=BD=9C?= =?UTF-8?q?=E4=B8=BA=E9=BB=98=E8=AE=A4=E5=91=BD=E4=BB=A4=E8=A1=8C=EF=BC=8C?= =?UTF-8?q?=E8=80=8C=E9=9D=9Epython=20startup.py=20=20=201.=20=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E6=96=B0=E7=9A=84METAGPT=5FROOT=E7=94=9F=E6=88=90?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=EF=BC=8C=E8=80=8C=E9=9D=9E=E5=AF=BB=E6=89=BE?= =?UTF-8?q?git=EF=BC=8C=E4=BB=A5=E4=BE=BFcli=E5=AE=89=E8=A3=85=20=20=202.?= =?UTF-8?q?=20=E5=91=BD=E4=BB=A4=E8=A1=8C=E7=94=B1fire=E6=8D=A2=E4=B8=BA?= =?UTF-8?q?=E4=BA=86typer=EF=BC=8C=E5=AE=83=E4=BC=9A=E5=B8=A6=E6=9D=A5?= =?UTF-8?q?=E7=9B=B8=E5=AF=B9=E6=9B=B4=E5=A5=BD=E7=9A=84=E4=BD=93=E9=AA=8C?= =?UTF-8?q?=20=20=203.=20project=5Fname=E5=8F=AF=E4=BB=A5=E7=81=B5?= =?UTF-8?q?=E6=B4=BB=E6=8C=87=E5=AE=9A=E4=BA=86=EF=BC=8C=E5=9C=A8metagpt?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E8=A1=8C=E8=BE=93=E5=85=A5=E4=B8=AD=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=205.=20=E5=85=B6=E4=BB=96=20=20=201.=20BossRequiremen?= =?UTF-8?q?t=20->=20UserRequirement=20=20=202.=20=E5=A4=A7=E9=87=8F?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E6=96=87=E6=9C=AC=E7=9A=84=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E4=BA=86=E5=8F=AF=E8=AF=BB=E6=80=A7?= =?UTF-8?q?=20=20=203.=20=E4=B8=AD=E9=87=8F=E6=8F=90=E7=A4=BA=E8=AF=8D?= =?UTF-8?q?=E4=BC=98=E5=8C=96=EF=BC=8C=E7=A8=8D=E5=BE=AE=E6=8F=90=E5=8D=87?= =?UTF-8?q?=E4=BA=86=E4=B8=80=E4=BA=9B=E5=87=86=E7=A1=AE=E7=8E=87=20=20=20?= =?UTF-8?q?4.=20=E6=9A=82=E6=97=B6=E5=B1=8F=E8=94=BD=E4=BA=86LongtermMemor?= =?UTF-8?q?y=E7=9B=B8=E5=85=B3=E9=80=BB=E8=BE=91=EF=BC=8C=E8=BF=99?= =?UTF-8?q?=E4=B8=AA=E9=80=BB=E8=BE=91=E5=BA=95=E5=B1=82=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E4=BA=86langchain=E7=9A=84FAISS=EF=BC=8C=E4=BC=9A=E5=B8=A6?= =?UTF-8?q?=E6=9D=A5~5=E7=A7=92=E5=8A=A0=E8=BD=BD=E8=80=97=E6=97=B6=20=20?= =?UTF-8?q?=205.=20=E4=BF=AE=E5=A4=8D=E4=BA=86=E5=AE=89=E8=A3=85=E5=8C=85?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E9=83=A8=E5=88=86=E6=8F=8F=E8=BF=B0=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/agent_creator.py | 7 +- examples/debate.py | 4 +- examples/sk_agent.py | 10 +- metagpt/actions/SummarizeCode.py | 93 ++++++++ metagpt/actions/__init__.py | 4 +- metagpt/actions/add_requirement.py | 4 +- metagpt/actions/design_api.py | 35 ++- metagpt/actions/project_management.py | 13 +- metagpt/actions/write_code.py | 19 +- metagpt/actions/write_code_review.py | 18 +- metagpt/actions/write_prd.py | 29 +-- metagpt/actions/write_test.py | 2 +- metagpt/config.py | 19 +- metagpt/const.py | 74 ++++--- metagpt/document.py | 207 ++++++++++++++++++ metagpt/document_store/base_store.py | 10 +- metagpt/document_store/document.py | 82 ------- metagpt/document_store/faiss_store.py | 10 +- metagpt/document_store/repo_parser.py | 90 ++++++++ metagpt/environment.py | 12 +- metagpt/logs.py | 10 +- metagpt/manager.py | 2 +- metagpt/memory/__init__.py | 4 +- metagpt/roles/engineer.py | 23 +- metagpt/roles/product_manager.py | 4 +- metagpt/roles/qa_engineer.py | 9 +- metagpt/roles/role.py | 5 +- metagpt/roles/sk_agent.py | 4 +- metagpt/software_company.py | 13 -- metagpt/startup.py | 45 ++++ metagpt/team.py | 18 +- metagpt/tools/sd_engine.py | 14 +- metagpt/utils/mermaid.py | 8 +- metagpt/utils/token_counter.py | 7 +- setup.py | 11 +- startup.py | 72 ------ tests/metagpt/actions/mock.py | 4 +- tests/metagpt/actions/test_write_prd.py | 4 +- tests/metagpt/document_store/test_document.py | 16 +- tests/metagpt/memory/test_longterm_memory.py | 12 +- tests/metagpt/memory/test_memory_storage.py | 10 +- tests/metagpt/planner/test_action_planner.py | 4 +- tests/metagpt/planner/test_basic_planner.py | 4 +- tests/metagpt/roles/mock.py | 8 +- tests/metagpt/roles/ui_role.py | 7 +- tests/metagpt/test_environment.py | 4 +- tests/metagpt/tools/test_sd_tool.py | 6 +- tests/metagpt/utils/test_common.py | 6 +- tests/metagpt/utils/test_output_parser.py | 2 +- tests/metagpt/utils/test_read_docx.py | 4 +- 50 files changed, 697 insertions(+), 385 deletions(-) create mode 100644 metagpt/actions/SummarizeCode.py create mode 100644 metagpt/document.py delete mode 100644 metagpt/document_store/document.py create mode 100644 metagpt/document_store/repo_parser.py delete mode 100644 metagpt/software_company.py create mode 100644 metagpt/startup.py delete mode 100644 startup.py diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 325e7c260..bcb9c0c1d 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -5,13 +5,14 @@ ''' import re -from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT +from metagpt.const import METAGPT_ROOT +from metagpt.config import CONFIG from metagpt.actions import Action from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger -with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: +with open(METAGPT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() @@ -49,7 +50,7 @@ def parse_code(rsp): pattern = r'```python(.*)```' match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" - with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: + with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text diff --git a/examples/debate.py b/examples/debate.py index a37e60848..0f5d1591b 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -8,7 +8,7 @@ import fire from metagpt.team import Team -from metagpt.actions import Action, BossRequirement +from metagpt.actions import Action, UserRequirement from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger @@ -49,7 +49,7 @@ def __init__( ): super().__init__(name, profile, **kwargs) self._init_actions([SpeakAloud]) - self._watch([BossRequirement, SpeakAloud]) + self._watch([UserRequirement, SpeakAloud]) self.name = name self.opponent_name = opponent_name diff --git a/examples/sk_agent.py b/examples/sk_agent.py index a7513e838..647ea4380 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -13,7 +13,7 @@ # from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -39,7 +39,7 @@ async def basic_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def sequential_planner_example(): @@ -53,7 +53,7 @@ async def sequential_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def basic_planner_web_search_example(): @@ -64,7 +64,7 @@ async def basic_planner_web_search_example(): role.import_skill(SkSearchEngine(), "WebSearchSkill") # role.import_semantic_skill_from_directory(skills_directory, "QASkill") - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def action_planner_example(): @@ -75,7 +75,7 @@ async def action_planner_example(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add + await role.run(Message(content=task, cause_by=UserRequirement)) # it will choose mathskill.Add if __name__ == "__main__": diff --git a/metagpt/actions/SummarizeCode.py b/metagpt/actions/SummarizeCode.py new file mode 100644 index 000000000..1015d3bfb --- /dev/null +++ b/metagpt/actions/SummarizeCode.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Author : alexanderwu +@File : SummarizeCode.py +""" + +from metagpt.actions.action import Action +from metagpt.logs import logger +from metagpt.schema import Message +from metagpt.utils.common import CodeParser +from tenacity import retry, stop_after_attempt, wait_fixed + +PROMPT_TEMPLATE = """ +NOTICE +Role: You are a professional software engineer, and your main task is to review the code. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". + +----- +# Context +{context} +----- + +## Code Review All: 请你对历史所有文件进行阅读，分析每个文件是否都完整实现了用户需求，找到可能的bug，如函数未实现、调用错误、未引用等 + +## Summary: 根据历史文件的实现情况进行总结 + +## Call flow: 根据实现的函数，使用mermaid绘制完整的调用链 + +## TODOs: 这里写出需要修改的文件列表，我们会在之后进行修改 + +""" + +FORMAT_EXAMPLE = """ + +## Code Review All + +### a.py +- 它少实现了xxx需求... +- 字段yyy没有给出... +- ... + +### b.py +... + +### c.py +... + +## Call flow +```mermaid +flowchart TB + c1-->a2 + subgraph one + a1-->a2 + end + subgraph two + b1-->b2 + end + subgraph three + c1-->c2 + end +``` + +## Summary +- a.py:... +- b.py:... +- c.py:... +- ... + +## TODOs +1. ... +2. ... +3. ... + +""" + + +class SummarizeCode(Action): + def __init__(self, name="SummaryCode", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + async def write_code_review_all(self, prompt): + code_rsp = await self._aask(prompt) + return code_rsp + + async def run(self, context): + format_example = FORMAT_EXAMPLE.format() + prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) + logger.info(f'Code review all..') + rsp = await self.write_code_review_all(prompt) + return rsp + \ No newline at end of file diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index b004bd58e..79ff94b3e 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -9,7 +9,7 @@ from metagpt.actions.action import Action from metagpt.actions.action_output import ActionOutput -from metagpt.actions.add_requirement import BossRequirement +from metagpt.actions.add_requirement import UserRequirement from metagpt.actions.debug_error import DebugError from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview @@ -28,7 +28,7 @@ class ActionType(Enum): """All types of Actions, used for indexing.""" - ADD_REQUIREMENT = BossRequirement + ADD_REQUIREMENT = UserRequirement WRITE_PRD = WritePRD WRITE_PRD_REVIEW = WritePRDReview WRITE_DESIGN = WriteDesign diff --git a/metagpt/actions/add_requirement.py b/metagpt/actions/add_requirement.py index 7dc09d062..8e2c56a62 100644 --- a/metagpt/actions/add_requirement.py +++ b/metagpt/actions/add_requirement.py @@ -8,7 +8,7 @@ from metagpt.actions import Action -class BossRequirement(Action): - """Boss Requirement without any implementation details""" +class UserRequirement(Action): + """User Requirement without any implementation details""" async def run(self, *args, **kwargs): raise NotImplementedError diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..f58d49495 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,6 @@ from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -27,21 +26,20 @@ ## Format example {format_example} ----- -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system Requirement: Fill in the following missing information based on the context, each section name is a key in json -Max Output: 8192 chars or 2048 tokens. Try to use them up. -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -52,7 +50,7 @@ "Implementation approach": "We will ...", "Python package name": "snake_game", "File list": ["main.py"], - "Data structures and interface definitions": ' + "Data structures and interfaces": ' classDiagram class Game{ +int score @@ -81,20 +79,19 @@ class Game{ ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Max Output: 8192 chars or 2048 tokens. Try to use them up. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ @@ -114,7 +111,7 @@ class Game{ ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Game{ @@ -143,7 +140,7 @@ class Game{ "Implementation approach": (str, ...), "Python package name": (str, ...), "File list": (List[str], ...), - "Data structures and interface definitions": (str, ...), + "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -177,8 +174,8 @@ async def _save_prd(self, docs_path, resources_path, context): async def _save_system_design(self, docs_path, resources_path, system_design): data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) + "Data structures and interfaces" + ] # CodeParser.parse_code(block="Data structures and interfaces", text=content) seq_flow = system_design.instruct_content.dict()[ "Program call flow" ] # CodeParser.parse_code(block="Program call flow", text=content) @@ -193,7 +190,7 @@ async def _save(self, context, system_design): ws_name = system_design.instruct_content.dict()["Python package name"] else: ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name + workspace = CONFIG.workspace_path / ws_name self.recreate_workspace(workspace) docs_path = workspace / "docs" resources_path = workspace / "resources" diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b395fa64e..467cb4d83 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -9,7 +9,6 @@ from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -27,9 +26,9 @@ Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Required Python third-party packages: Provided in requirements.txt format +## Required Python third-party packages: Provide Python list[str] in requirements.txt format -## Required Other language third-party packages: Provided in requirements.txt format +## Required Other language third-party packages: Provide Python list[str] in requirements.txt format ## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. @@ -39,7 +38,7 @@ ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -95,7 +94,7 @@ ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. """, "FORMAT_EXAMPLE": ''' @@ -171,11 +170,11 @@ def _save(self, context, rsp): ws_name = context[-1].instruct_content.dict()["Python package name"] else: ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" + file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" + requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) async def run(self, context, format=CONFIG.prompt_format): diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c000805c5..176718dfc 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,7 +7,7 @@ """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser @@ -18,19 +18,22 @@ Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". +----- +# Context +{context} +----- + ## Code: {filename} Write code with triple quoto, based on the following list and context. 1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. 2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. 5. Think before writing: What should be implemented and provided in this document? 6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. 7. Do not use public member functions that do not exist in your design. +8. Before using a variable, make sure you reference it first +9. Write out EVERY DETAIL, DON'T LEAVE TODO. ------ -# Context -{context} ------ ## Format example ----- ## Code: {filename} @@ -58,7 +61,7 @@ def _save(self, context, filename, code): design = [i for i in context if i.cause_by == WriteDesign][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name + ws_path = CONFIG.workspace_path / ws_name if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): ws_path = ws_path / ws_name code_path = ws_path / filename diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 4ff4d6cf6..c6538bf7b 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -17,16 +17,14 @@ Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -``` -1. Check 0: Is the code implemented as per the requirements? -2. Check 1: Are there any issues with the code logic? -3. Check 2: Does the existing code follow the "Data structures and interface definitions"? -4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? -5. Check 4: Does the code have unnecessary or lack dependencies? -``` +## Code Review: Based on the following context and code, follow the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. +1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. +2. Are there any issues with the code logic? If so, how to solve it? +3. Does the existing code follow the "Data structures and interfaces"? +4. Is there a function in the code that is not fully implemented? If so, how to implement it? +5. Does the code have unnecessary or lack dependencies? If so, how to solve it? -## Rewrite Code: {filename} Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE. +## Rewrite Code: rewrite {filename} based on "Code Review" with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. ----- # Context {context} @@ -47,7 +45,7 @@ FORMAT_EXAMPLE = """ ## Code Review -1. The code ... +1. No, we should add the logic of ... 2. ... 3. ... 4. ... diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..584d31998 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -46,24 +46,25 @@ {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design +Requirements: According to the context, fill in the following missing information, each section name is a key in json ## Original Requirements: Provide as Plain text, place the polished complete original requirements here -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible +## Competitive Analysis: Provided as Python list[str], up to 8 competitive product analyses ## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. +## Requirement Analysis: Provide as Plain text. -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards ## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. + +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -131,30 +132,30 @@ {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. ## Original Requirements: Provide as Plain text, place the polished complete original requirements here -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories ## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible ## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. +## Requirement Analysis: Provide as Plain text. -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards ## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ --- ## Original Requirements -The boss ... +The user ... ## Product Goals ```python diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 35ff36dc2..2f4988c09 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -15,7 +15,7 @@ 2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. 3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. 4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. +5. Attention3: YOU MUST FOLLOW "Data structures and interfaces". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. 6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? 7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. diff --git a/metagpt/config.py b/metagpt/config.py index 3f9e742bd..1a9cdb4d2 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -8,7 +8,9 @@ import openai import yaml -from metagpt.const import PROJECT_ROOT +from pathlib import Path + +from metagpt.const import METAGPT_ROOT, DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -35,13 +37,14 @@ class Config(metaclass=Singleton): """ _instance = None - key_yaml_file = PROJECT_ROOT / "config/key.yaml" - default_yaml_file = PROJECT_ROOT / "config/config.yaml" + home_yaml_file = Path.home() / ".metagpt/config.yaml" + key_yaml_file = METAGPT_ROOT / "config/key.yaml" + default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) - logger.info("Config loading done.") + # logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") @@ -94,12 +97,18 @@ def __init__(self, yaml_file=default_yaml_file): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) + self._ensure_workspace_exists() + + def _ensure_workspace_exists(self): + self.workspace_path.mkdir(parents=True, exist_ok=True) + logger.info(f"WORKSPACE_PATH set to {self.workspace_path}") def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" configs.update(os.environ) - for _yaml_file in [yaml_file, self.key_yaml_file]: + for _yaml_file in [yaml_file, self.key_yaml_file, self.home_yaml_file]: if not _yaml_file.exists(): continue diff --git a/metagpt/const.py b/metagpt/const.py index 407ce803a..14e692487 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -5,44 +5,54 @@ @Author : alexanderwu @File : const.py """ +import os from pathlib import Path from loguru import logger +import metagpt -def get_project_root(): - """Search upwards to find the project root directory.""" - current_path = Path.cwd() - while True: - if ( - (current_path / ".git").exists() - or (current_path / ".project_root").exists() - or (current_path / ".gitignore").exists() - ): - # use metagpt with git clone will land here - logger.info(f"PROJECT_ROOT set to {str(current_path)}") - return current_path - parent_path = current_path.parent - if parent_path == current_path: - # use metagpt with pip install will land here - cwd = Path.cwd() - logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") - return cwd - current_path = parent_path - - -PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / "data" -WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" -UT_PATH = PROJECT_ROOT / "data/ut" -SWAGGER_PATH = UT_PATH / "files/api/" -UT_PY_PATH = UT_PATH / "files/ut/" -API_QUESTIONS_PATH = UT_PATH / "files/question/" -YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / "tmp" + +def get_metagpt_package_root(): + """Get the root directory of the installed package.""" + package_root = Path(metagpt.__file__).parent.parent + logger.info(f"Package root set to {str(package_root)}") + return package_root + + +def get_metagpt_root(): + """Get the project root directory.""" + # Check if a project root is specified in the environment variable + project_root_env = os.getenv('METAGPT_PROJECT_ROOT') + if project_root_env: + project_root = Path(project_root_env) + logger.info(f"PROJECT_ROOT set from environment variable to {str(project_root)}") + else: + # Fallback to package root if no environment variable is set + project_root = get_metagpt_package_root() + return project_root + + +# METAGPT PROJECT ROOT AND VARS + +METAGPT_ROOT = get_metagpt_root() +DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace" + +DATA_PATH = METAGPT_ROOT / "data" RESEARCH_PATH = DATA_PATH / "research" TUTORIAL_PATH = DATA_PATH / "tutorial_docx" INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table" +UT_PATH = DATA_PATH / "ut" +SWAGGER_PATH = UT_PATH / "files/api/" +UT_PY_PATH = UT_PATH / "files/ut/" +API_QUESTIONS_PATH = UT_PATH / "files/question/" -SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" +TMP = METAGPT_ROOT / "tmp" + +SOURCE_ROOT = METAGPT_ROOT / "metagpt" +PROMPT_PATH = SOURCE_ROOT / "prompts" +SKILL_DIRECTORY = SOURCE_ROOT / "skills" + + +# REAL CONSTS MEM_TTL = 24 * 30 * 3600 +YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/document.py b/metagpt/document.py new file mode 100644 index 000000000..044210218 --- /dev/null +++ b/metagpt/document.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/6/8 14:03 +@Author : alexanderwu +@File : document.py +""" + +from typing import Union, Optional +from pathlib import Path +from pydantic import BaseModel, Field +import pandas as pd +from langchain.document_loaders import ( + TextLoader, + UnstructuredPDFLoader, + UnstructuredWordDocumentLoader, +) +from langchain.text_splitter import CharacterTextSplitter +from tqdm import tqdm + +from metagpt.logs import logger + + +def validate_cols(content_col: str, df: pd.DataFrame): + if content_col not in df.columns: + raise ValueError("Content column not found in DataFrame.") + + +def read_data(data_path: Path): + suffix = data_path.suffix + if '.xlsx' == suffix: + data = pd.read_excel(data_path) + elif '.csv' == suffix: + data = pd.read_csv(data_path) + elif '.json' == suffix: + data = pd.read_json(data_path) + elif suffix in ('.docx', '.doc'): + data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() + elif '.txt' == suffix: + data = TextLoader(str(data_path)).load() + text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) + texts = text_splitter.split_documents(data) + data = texts + elif '.pdf' == suffix: + data = UnstructuredPDFLoader(str(data_path), mode="elements").load() + else: + raise NotImplementedError("File format not supported.") + return data + + +class Document(BaseModel): + """ + Document: Handles operations related to document files. + """ + content: str = Field(default='') + file_path: Path = Field(default=None) + + @classmethod + def from_path(cls, file_path: Path): + """ + Create a Document instance from a file path. + """ + if not file_path.exists(): + raise FileNotFoundError(f"File {file_path} not found.") + content = file_path.read_text() + return cls(content=content, file_path=file_path) + + @classmethod + def from_text(cls, text: str, file_path: Optional[Path] = None): + """ + Create a Document from a text string. + """ + return cls(content=text, file_path=file_path) + + def to_path(self, file_path: Optional[Path] = None): + """ + Save content to the specified file path. + """ + if file_path is not None: + self.file_path = file_path + + if self.file_path is None: + raise ValueError("File path is not set.") + + self.file_path.parent.mkdir(parents=True, exist_ok=True) + self.file_path.write_text(self.content) + + def persist(self): + """ + Persist document to disk. + """ + return self.to_path() + + +class IndexableDocument(Document): + """ + Advanced document handling: For vector databases or search engines. + """ + data: Union[pd.DataFrame, list] + content_col: Optional[str] = Field(default='') + meta_col: Optional[str] = Field(default='') + + class Config: + arbitrary_types_allowed = True + + @classmethod + def from_path(cls, data_path: Path, content_col='content', meta_col='metadata'): + if not data_path.exists(): + raise FileNotFoundError(f"File {data_path} not found.") + data = read_data(data_path) + content = data_path.read_text() + if isinstance(data, pd.DataFrame): + validate_cols(content_col, data) + return cls(data=data, content=content, content_col=content_col, meta_col=meta_col) + + def _get_docs_and_metadatas_by_df(self) -> (list, list): + df = self.data + docs = [] + metadatas = [] + for i in tqdm(range(len(df))): + docs.append(df[self.content_col].iloc[i]) + if self.meta_col: + metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) + else: + metadatas.append({}) + return docs, metadatas + + def _get_docs_and_metadatas_by_langchain(self) -> (list, list): + data = self.data + docs = [i.page_content for i in data] + metadatas = [i.metadata for i in data] + return docs, metadatas + + def get_docs_and_metadatas(self) -> (list, list): + if isinstance(self.data, pd.DataFrame): + return self._get_docs_and_metadatas_by_df() + elif isinstance(self.data, list): + return self._get_docs_and_metadatas_by_langchain() + else: + raise NotImplementedError("Data type not supported for metadata extraction.") + + +class Repo(BaseModel): + + # Name of this repo. + name: str = Field(default="") + docs: dict[Path, Document] = Field(default_factory=dict) + codes: dict[Path, Document] = Field(default_factory=dict) + assets: dict[Path, Document] = Field(default_factory=dict) + repo_path: Path = Field(default_factory=Path) + + def _path(self, filename): + return self.repo_path / filename + + @classmethod + def from_path(cls, repo_path: Path): + """Load documents, code, and assets from a repository path.""" + repo_path.mkdir(parents=True, exist_ok=True) + repo = Repo(repo_path = repo_path) + for file_path in repo_path.rglob('*'): + if file_path.is_file(): + repo._set(file_path.read_text(), file_path) + return repo + + def to_path(self): + """Persist all documents, code, and assets to the given repository path.""" + for doc in self.docs.values(): + doc.to_path() + for code in self.codes.values(): + code.to_path() + for asset in self.assets.values(): + asset.to_path() + + def _set(self, content: str, file_path: Path): + """Add a document to the appropriate category based on its file extension.""" + file_ext = file_path.suffix + + doc = Document(content=content, file_path=file_path) + if file_ext.lower() == '.md': + self.docs[file_path] = doc + elif file_ext.lower() in ['.py', '.js', '.css', '.html']: + self.codes[file_path] = doc + else: + self.assets[file_path] = doc + return doc + + def set(self, content: str, filename: str): + """Set a document and persist it to disk.""" + file_path = self._path(filename) + doc = self._set(content, file_path) + doc.to_path() + + def get(self, filename: str) -> Optional[Document]: + """Get a document by its filename.""" + path = self._path(filename) + return self.docs.get(path) or self.codes.get(path) or self.assets.get(path) + + +def main(): + repo1 = Repo.from_path(Path("/Users/alexanderwu/workspace/t1")) + repo1.set("wtf content", "doc/wtf_file.md") + repo1.set("wtf code", "code/wtf_file.py") + logger.info(repo1) # check doc + + +if __name__ == '__main__': + main() diff --git a/metagpt/document_store/base_store.py b/metagpt/document_store/base_store.py index 5d7015e8b..84b47a98c 100644 --- a/metagpt/document_store/base_store.py +++ b/metagpt/document_store/base_store.py @@ -28,20 +28,20 @@ def add(self, *args, **kwargs): class LocalStore(BaseStore, ABC): - def __init__(self, raw_data: Path, cache_dir: Path = None): - if not raw_data: + def __init__(self, raw_data_path: Path, cache_dir: Path = None): + if not raw_data_path: raise FileNotFoundError self.config = Config() - self.raw_data = raw_data + self.raw_data_path = raw_data_path if not cache_dir: - cache_dir = raw_data.parent + cache_dir = raw_data_path.parent self.cache_dir = cache_dir self.store = self._load() if not self.store: self.store = self.write() def _get_index_and_store_fname(self): - fname = self.raw_data.name.split('.')[0] + fname = self.raw_data_path.name.split('.')[0] index_file = self.cache_dir / f"{fname}.index" store_file = self.cache_dir / f"{fname}.pkl" return index_file, store_file diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py deleted file mode 100644 index e4b9473c7..000000000 --- a/metagpt/document_store/document.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/6/8 14:03 -@Author : alexanderwu -@File : document.py -""" -from pathlib import Path - -import pandas as pd -from langchain.document_loaders import ( - TextLoader, - UnstructuredPDFLoader, - UnstructuredWordDocumentLoader, -) -from langchain.text_splitter import CharacterTextSplitter -from tqdm import tqdm - - -def validate_cols(content_col: str, df: pd.DataFrame): - if content_col not in df.columns: - raise ValueError - - -def read_data(data_path: Path): - suffix = data_path.suffix - if '.xlsx' == suffix: - data = pd.read_excel(data_path) - elif '.csv' == suffix: - data = pd.read_csv(data_path) - elif '.json' == suffix: - data = pd.read_json(data_path) - elif suffix in ('.docx', '.doc'): - data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() - elif '.txt' == suffix: - data = TextLoader(str(data_path)).load() - text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) - texts = text_splitter.split_documents(data) - data = texts - elif '.pdf' == suffix: - data = UnstructuredPDFLoader(str(data_path), mode="elements").load() - else: - raise NotImplementedError - return data - - -class Document: - - def __init__(self, data_path, content_col='content', meta_col='metadata'): - self.data = read_data(data_path) - if isinstance(self.data, pd.DataFrame): - validate_cols(content_col, self.data) - self.content_col = content_col - self.meta_col = meta_col - - def _get_docs_and_metadatas_by_df(self) -> (list, list): - df = self.data - docs = [] - metadatas = [] - for i in tqdm(range(len(df))): - docs.append(df[self.content_col].iloc[i]) - if self.meta_col: - metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) - else: - metadatas.append({}) - - return docs, metadatas - - def _get_docs_and_metadatas_by_langchain(self) -> (list, list): - data = self.data - docs = [i.page_content for i in data] - metadatas = [i.metadata for i in data] - return docs, metadatas - - def get_docs_and_metadatas(self) -> (list, list): - if isinstance(self.data, pd.DataFrame): - return self._get_docs_and_metadatas_by_df() - elif isinstance(self.data, list): - return self._get_docs_and_metadatas_by_langchain() - else: - raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index dd450010d..885ad3e15 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -15,15 +15,15 @@ from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore -from metagpt.document_store.document import Document +from metagpt.document import IndexableDocument from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'): + def __init__(self, raw_data_path: Path, cache_dir=None, meta_col='source', content_col='output'): self.meta_col = meta_col self.content_col = content_col - super().__init__(raw_data, cache_dir) + super().__init__(raw_data_path, cache_dir) def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() @@ -60,9 +60,9 @@ def search(self, query, expand_cols=False, sep='\n', *args, k=5, **kwargs): def write(self): """Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user.""" - if not self.raw_data.exists(): + if not self.raw_data_path.exists(): raise FileNotFoundError - doc = Document(self.raw_data, self.content_col, self.meta_col) + doc = IndexableDocument.from_path(self.raw_data_path, self.content_col, self.meta_col) docs, metadatas = doc.get_docs_and_metadatas() self.store = self._write(docs, metadatas) diff --git a/metagpt/document_store/repo_parser.py b/metagpt/document_store/repo_parser.py new file mode 100644 index 000000000..f7e2b0f4a --- /dev/null +++ b/metagpt/document_store/repo_parser.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/17 17:58 +@Author : alexanderwu +@File : repo_parser.py +""" +import json +import pathlib +import ast + +import pandas as pd + + +class RepoParser: + def __init__(self): + self.base_directory = None + + def parse_file(self, file_path): + """Parse a Python file in the repository.""" + try: + return ast.parse(file_path.read_text()).body + except: + return [] + + def extract_class_and_function_info(self, tree, file_path): + """Extract class, function, and global variable information from the AST.""" + file_info = { + "file": str(file_path.relative_to(self.base_directory)), + "classes": [], + "functions": [], + "globals": [] + } + + for node in tree: + if isinstance(node, ast.ClassDef): + class_methods = [m.name for m in node.body if is_func(m)] + file_info["classes"].append({"name": node.name, "methods": class_methods}) + elif is_func(node): + file_info["functions"].append(node.name) + elif isinstance(node, ast.Assign) or isinstance(node, ast.AnnAssign): + for target in node.targets if isinstance(node, ast.Assign) else [node.target]: + if isinstance(target, ast.Name): + file_info["globals"].append(target.id) + return file_info + + def generate_json_structure(self, directory, output_path): + """Generate a JSON file documenting the repository structure.""" + files_classes = [] + for path in directory.rglob('*.py'): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + output_path.write_text(json.dumps(files_classes, indent=4)) + + def generate_dataframe_structure(self, directory, output_path): + """Generate a DataFrame documenting the repository structure and save as CSV.""" + files_classes = [] + for path in directory.rglob('*.py'): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + df = pd.DataFrame(files_classes) + df.to_csv(output_path, index=False) + + def generate_structure(self, directory_path, output_path=None, mode='json'): + """Generate the structure of the repository as a specified format.""" + self.base_directory = pathlib.Path(directory_path) + output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}" + output_path = pathlib.Path(output_path) if output_path else output_file + + if mode == 'json': + self.generate_json_structure(self.base_directory, output_path) + elif mode == 'csv': + self.generate_dataframe_structure(self.base_directory, output_path) + + +def is_func(node): + return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + + +def main(): + repo_parser = RepoParser() + repo_parser.generate_structure("/Users/alexanderwu/git/mg1/metagpt", "/Users/alexanderwu/git/mg1/mg1-structure.csv", mode='csv') + + +if __name__ == '__main__': + main() diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..38077c90d 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -10,20 +10,22 @@ from pydantic import BaseModel, Field +# from metagpt.document import Document +from metagpt.document import Repo from metagpt.memory import Memory from metagpt.roles import Role from metagpt.schema import Message class Environment(BaseModel): - """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 - Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - + """ + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles """ roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) history: str = Field(default='') + repo: Repo = Field(default_factory=Repo) class Config: arbitrary_types_allowed = True @@ -50,6 +52,10 @@ def publish_message(self, message: Message): self.memory.add(message) self.history += f"\n{message}" + def publish_doc(self, content: str, filename: str): + """向当前环境发布文档（包括代码）""" + self.repo.set(content, filename) + async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once diff --git a/metagpt/logs.py b/metagpt/logs.py index b2052e9b8..afebbfed9 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -10,15 +10,15 @@ from loguru import logger as _logger -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT + def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """调整日志级别到level之上 - Adjust the log level to above level - """ + """Adjust the log level to above level""" _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) + _logger.add(METAGPT_ROOT / 'logs/log.txt', level=logfile_level) return _logger + logger = define_log_level() diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..7cbbe651e 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -14,7 +14,7 @@ class Manager: def __init__(self, llm: LLM = LLM()): self.llm = llm # Large Language Model self.role_directions = { - "BOSS": "Product Manager", + "User": "Product Manager", "Product Manager": "Architect", "Architect": "Engineer", "Engineer": "QA Engineer", diff --git a/metagpt/memory/__init__.py b/metagpt/memory/__init__.py index 710930626..bd6e72163 100644 --- a/metagpt/memory/__init__.py +++ b/metagpt/memory/__init__.py @@ -7,10 +7,10 @@ """ from metagpt.memory.memory import Memory -from metagpt.memory.longterm_memory import LongTermMemory +# from metagpt.memory.longterm_memory import LongTermMemory __all__ = [ "Memory", - "LongTermMemory", + # "LongTermMemory", ] diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 1f6685b38..171af47f0 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,7 +11,8 @@ from pathlib import Path from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.SummarizeCode import SummarizeCode +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -80,13 +81,13 @@ def __init__( self.n_borg = n_borg @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: + def parse_tasks(cls, task_msg: Message) -> list[str]: if task_msg.instruct_content: return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) @classmethod - def parse_code(self, code_text: str) -> str: + def parse_code(cls, code_text: str) -> str: return CodeParser.parse_code(block="", text=code_text) @classmethod @@ -98,10 +99,10 @@ def parse_workspace(cls, system_design_msg: Message) -> str: def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def recreate_workspace(self): workspace = self.get_workspace() @@ -167,7 +168,7 @@ async def _act_sp(self) -> Message: ) return msg - async def _act_sp_precision(self) -> Message: + async def _act_sp_with_cr(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: """ @@ -191,7 +192,6 @@ async def _act_sp_precision(self) -> Message: code = rewrite_code except Exception as e: logger.error("code review failed!", e) - pass file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -199,6 +199,13 @@ async def _act_sp_precision(self) -> Message: code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) + context = [] + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + for m in msg: + context.append(m.content) + context_str = "\n".join(context) + code_review_all = await SummarizeCode().run(context=context_str) + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" @@ -209,5 +216,5 @@ async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: - return await self._act_sp_precision() + return await self._act_sp_with_cr() return await self._act_sp() diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..f6172b607 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : product_manager.py """ -from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions import UserRequirement, WritePRD from metagpt.roles import Role @@ -38,4 +38,4 @@ def __init__( """ super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a763c2ce8..f124646b3 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -16,7 +16,8 @@ WriteDesign, WriteTest, ) -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -50,13 +51,13 @@ def parse_workspace(cls, system_design_msg: Message) -> str: def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: - return WORKSPACE_ROOT / workspace + return CONFIG.workspace_path / workspace # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def write_file(self, filename: str, code: str): workspace = self.get_workspace() / "tests" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b96c361c0..d772c0748 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -17,7 +17,8 @@ from metagpt.actions import Action, ActionOutput from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import Memory +# from metagpt.memory import LongTermMemory from metagpt.schema import Message PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -78,7 +79,7 @@ class RoleContext(BaseModel): """Role Runtime Context""" env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) - long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) + # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index b27841d74..4069f4836 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -9,7 +9,7 @@ from semantic_kernel.planning.action_planner.action_planner import ActionPlanner from semantic_kernel.planning.basic_planner import BasicPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions.execute_task import ExecuteTask from metagpt.logs import logger from metagpt.roles import Role @@ -39,7 +39,7 @@ def __init__( """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([ExecuteTask()]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) self.kernel = make_sk_kernel() # how funny the interface is inconsistent diff --git a/metagpt/software_company.py b/metagpt/software_company.py deleted file mode 100644 index d44a0068a..000000000 --- a/metagpt/software_company.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/12 00:30 -@Author : alexanderwu -@File : software_company.py -""" -from metagpt.team import Team as SoftwareCompany - -import warnings -warnings.warn("metagpt.software_company is deprecated and will be removed in the future" - "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", - DeprecationWarning, 2) diff --git a/metagpt/startup.py b/metagpt/startup.py new file mode 100644 index 000000000..d8ca4072f --- /dev/null +++ b/metagpt/startup.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from pathlib import Path +import asyncio +import typer + +app = typer.Typer() + + +@app.command() +def startup( + idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"), + investment: float = typer.Option(3.0, help="Dollar amount to invest in the AI company."), + n_round: int = typer.Option(5, help="Number of rounds for the simulation."), + code_review: bool = typer.Option(True, help="Whether to use code review."), + run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), + implement: bool = typer.Option(True, help="Enable or disable code implementation."), + project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'"), +): + """Run a startup. Be a boss.""" + from metagpt.roles import ProductManager, Architect, ProjectManager, Engineer, QaEngineer + from metagpt.team import Team + + company = Team() + company.hire( + [ + ProductManager(), + Architect(), + ProjectManager(), + ] + ) + + if implement or code_review: + company.hire([Engineer(n_borg=5, use_code_review=code_review)]) + + if run_tests: + company.hire([QaEngineer()]) + + company.invest(investment) + company.start_project(project_name, idea) + asyncio.run(company.run(n_round=n_round)) + + +if __name__ == "__main__": + app() diff --git a/metagpt/team.py b/metagpt/team.py index 67d3ecec8..2332aaa46 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -7,7 +7,7 @@ """ from pydantic import BaseModel, Field -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger @@ -21,7 +21,7 @@ class Team(BaseModel): Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to perform any multi-agent activity, such as collaboratively writing executable code. """ - environment: Environment = Field(default_factory=Environment) + env: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -30,7 +30,7 @@ class Config: def hire(self, roles: list[Role]): """Hire roles to cooperate""" - self.environment.add_roles(roles) + self.env.add_roles(roles) def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" @@ -42,10 +42,12 @@ def _check_balance(self): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea, send_to: str = ""): - """Start a project from publishing boss requirement.""" + def start_project(self, project_name, idea, send_to: str = ""): + """Start a project from publishing user requirement.""" self.idea = idea - self.environment.publish_message(Message(role="Human", content=idea, cause_by=BossRequirement, send_to=send_to)) + # If user set project_name, then use it. + self.env.repo.name = project_name + self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) def _save(self): logger.info(self.json()) @@ -57,6 +59,6 @@ async def run(self, n_round=3): n_round -= 1 logger.debug(f"{n_round=}") self._check_balance() - await self.environment.run() - return self.environment.history + await self.env.run() + return self.env.history \ No newline at end of file diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 1d9cd0b2a..4f010a912 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -13,12 +13,10 @@ from aiohttp import ClientSession from PIL import Image, PngImagePlugin -from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG +# from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -config = Config() - payload = { "prompt": "", "negative_prompt": "(easynegative:0.8),black, dark,Low resolution", @@ -56,9 +54,8 @@ class SDEngine: def __init__(self): # Initialize the SDEngine with configuration - self.config = Config() - self.sd_url = self.config.get("SD_URL") - self.sd_t2i_url = f"{self.sd_url}{self.config.get('SD_T2I_API')}" + self.sd_url = CONFIG.get("SD_URL") + self.sd_t2i_url = f"{self.sd_url}{CONFIG.get('SD_T2I_API')}" # Define default payload settings for SD API self.payload = payload logger.info(self.sd_t2i_url) @@ -81,7 +78,7 @@ def construct_payload( return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.workspace_path / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) @@ -125,6 +122,7 @@ def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): save_name = join(save_dir, save_name) decode_base64_to_image(_img, save_name=save_name) + if __name__ == "__main__": engine = SDEngine() prompt = "pixel style, game design, a game interface should be minimalistic and intuitive with the score and high score displayed at the top. The snake and its food should be easily distinguishable. The game should have a simple color scheme, with a contrasting color for the snake and its food. Complete interface boundary" diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 204c22c67..eb85a3f90 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -10,7 +10,7 @@ from pathlib import Path from metagpt.config import CONFIG -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists @@ -69,7 +69,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, if stdout: logger.info(stdout.decode()) if stderr: - logger.error(stderr.decode()) + logger.warning(stderr.decode()) else: if engine == "playwright": from metagpt.utils.mmdc_playwright import mermaid_to_file @@ -141,6 +141,6 @@ class KnowledgeBase { if __name__ == "__main__": loop = asyncio.new_event_loop() - result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) - result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) loop.close() diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 1af96f272..33bcd01a5 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -21,6 +21,7 @@ "gpt-4-32k": {"prompt": 0.06, "completion": 0.12}, "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12}, "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, + "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -37,6 +38,7 @@ "gpt-4-32k": 32768, "gpt-4-32k-0314": 32768, "gpt-4-0613": 8192, + "gpt-4-1106-preview": 128000, "text-embedding-ada-002": 8192, "chatglm_turbo": 32768 } @@ -56,16 +58,17 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613", + "gpt-4-1106-preview", }: tokens_per_message = 3 tokens_per_name = 1 elif model == "gpt-3.5-turbo-0301": tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n tokens_per_name = -1 # if there's a name, the role is omitted - elif "gpt-3.5-turbo" in model: + elif "gpt-3.5-turbo" == model: print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.") return count_message_tokens(messages, model="gpt-3.5-turbo-0613") - elif "gpt-4" in model: + elif "gpt-4" == model: print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.") return count_message_tokens(messages, model="gpt-4-0613") else: diff --git a/setup.py b/setup.py index 239156ae3..e7462767f 100644 --- a/setup.py +++ b/setup.py @@ -31,14 +31,14 @@ def run(self): setup( name="metagpt", version="0.3.0", - description="The Multi-Role Meta Programming Framework", + description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@fuzhi.ai", - license="Apache 2.0", - keywords="metagpt multi-role multi-agent programming gpt llm", + license="MIT", + keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming", packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, @@ -52,4 +52,9 @@ def run(self): cmdclass={ "install_mermaid": InstallMermaidCLI, }, + entry_points={ + 'console_scripts': [ + 'metagpt=metagpt.startup:app', + ], + }, ) diff --git a/startup.py b/startup.py deleted file mode 100644 index e9fbf94d3..000000000 --- a/startup.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import asyncio - -import fire - -from metagpt.roles import ( - Architect, - Engineer, - ProductManager, - ProjectManager, - QaEngineer, -) -from metagpt.team import Team - - -async def startup( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = False, - run_tests: bool = False, - implement: bool = True, -): - """Run a startup. Be a boss.""" - company = Team() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) - - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) - - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) - - -def main( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = True, - run_tests: bool = False, - implement: bool = True, -): - """ - We are a software startup comprised of AI. By investing in us, - you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." - :param investment: As an investor, you have the opportunity to contribute - a certain dollar amount to this AI company. - :param n_round: - :param code_review: Whether to use code review. - :return: - """ - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) - - -if __name__ == "__main__": - fire.Fire(main) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index a800690e8..5be1d8001 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -100,7 +100,7 @@ file_list = ["main.py", "room.py", "player.py", "game.py", "object.py", "puzzle.py", "test_game.py"] ``` -## Data structures and interface definitions: +## Data structures and interfaces: ```mermaid classDiagram class Room{ @@ -209,7 +209,7 @@ class Game{ """ ``` -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. ```python """ The original requirements did not specify whether the game should have a save/load feature, multiplayer support, or any specific graphical user interface. More information on these aspects could help in further refining the product design and requirements. diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 38e4e5221..18675ecc3 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -7,7 +7,7 @@ """ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message @@ -17,7 +17,7 @@ async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement)) + prd = await product_manager.handle(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) diff --git a/tests/metagpt/document_store/test_document.py b/tests/metagpt/document_store/test_document.py index 5ae357fb1..13c0921a3 100644 --- a/tests/metagpt/document_store/test_document.py +++ b/tests/metagpt/document_store/test_document.py @@ -7,22 +7,22 @@ """ import pytest -from metagpt.const import DATA_PATH -from metagpt.document_store.document import Document +from metagpt.const import METAGPT_ROOT +from metagpt.document import IndexableDocument CASES = [ - ("st/faq.xlsx", "Question", "Answer", 1), - ("cases/faq.csv", "Question", "Answer", 1), + ("requirements.txt", None, None, 0), + # ("cases/faq.csv", "Question", "Answer", 1), # ("cases/faq.json", "Question", "Answer", 1), - ("docx/faq.docx", None, None, 1), - ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 - ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 + # ("docx/faq.docx", None, None, 1), + # ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 + # ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 ] @pytest.mark.parametrize("relative_path, content_col, meta_col, threshold", CASES) def test_document(relative_path, content_col, meta_col, threshold): - doc = Document(DATA_PATH / relative_path, content_col, meta_col) + doc = IndexableDocument.from_path(METAGPT_ROOT / relative_path, content_col, meta_col) rsp = doc.get_docs_and_metadatas() assert len(rsp[0]) > threshold assert len(rsp[1]) > threshold diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index dc5540520..ac9362937 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -4,7 +4,7 @@ from metagpt.config import CONFIG from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.role import RoleContext from metagpt.memory import LongTermMemory @@ -15,24 +15,24 @@ def test_ltm_search(): assert len(openai_api_key) > 20 role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + rc = RoleContext(watch=[UserRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -48,7 +48,7 @@ def test_ltm_search(): assert len(news) == 0 new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 6bb3e8f1d..bd4441641 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -6,7 +6,7 @@ from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions import WritePRD from metagpt.actions.action_output import ActionOutput @@ -14,7 +14,7 @@ def test_idea_message(): idea = 'Write a cli snake game' role_id = 'UTUser1(Product Manager)' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -24,12 +24,12 @@ def test_idea_message(): assert memory_storage.is_initialized is True sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) new_messages = memory_storage.search(sim_message) assert len(new_messages) == 0 # similar, return [] new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -49,7 +49,7 @@ def test_actionout_message(): ic_obj = ActionOutput.create_model_class('prd', out_mapping) role_id = 'UTUser2(Architect)' - content = 'The boss has requested the creation of a command-line interface (CLI) snake game' + content = 'The user has requested the creation of a command-line interface (CLI) snake game' message = Message(content=content, instruct_content=ic_obj(**out_data), role='user', diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index 5ab9a493f..8efe6cfc4 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -9,7 +9,7 @@ from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -23,7 +23,7 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 03a82ec5e..f6d44ba03 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -8,7 +8,7 @@ import pytest from semantic_kernel.core_skills import TextSkill -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -26,7 +26,7 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate assert "WriterSkill.Brainstorm" in role.plan.generated_plan.result diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 52fc4a3c1..fbad06acb 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -5,10 +5,10 @@ @Author : alexanderwu @File : mock.py """ -from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks +from metagpt.actions import UserRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message -BOSS_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" +USER_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" DETAIL_REQUIREMENT = """需求：开发一个基于LLM（大语言模型）与私有知识库的搜索引擎，希望有几点能力 1. 用户可以在私有知识库进行搜索，再根据大语言模型进行总结，输出的结果包括了总结 @@ -94,7 +94,7 @@ ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Main { @@ -252,7 +252,7 @@ class KnowledgeBase { class MockMessages: - req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement) + req = Message(role="User", content=USER_REQUIREMENT, cause_by=UserRequirement) prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD) system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign) tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..102c6ebd6 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -8,7 +8,8 @@ from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -29,7 +30,7 @@ ## Selected Elements:Provide as Plain text, up to 5 specified elements, clear and simple ## HTML Layout:Provide as Plain text, use standard HTML code ## CSS Styles (styles.css):Provide as Plain text,use standard css code -## Anything UNCLEAR:Provide as Plain text. Make clear here. +## Anything UNCLEAR:Provide as Plain text. Try to clarify it. """ @@ -214,7 +215,7 @@ async def draw_icons(self, context, *args, **kwargs): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace_path / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..b27bc3da7 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -8,7 +8,7 @@ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.environment import Environment from metagpt.logs import logger from metagpt.manager import Manager @@ -49,7 +49,7 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) env.set_manager(Manager()) - env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) + env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) await env.run(k=2) logger.info(f"{env.history=}") diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..fea58bc29 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,9 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine + def test_sd_engine_init(): @@ -21,5 +23,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" + img_path = CONFIG.workspace_path / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) == True diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index ec4443175..b6c000f9b 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -10,7 +10,7 @@ import pytest -from metagpt.const import get_project_root +from metagpt.const import get_metagpt_root class TestGetProjectRoot: @@ -20,11 +20,11 @@ def change_etc_dir(self): os.chdir(abs_root) def test_get_project_root(self): - project_root = get_project_root() + project_root = get_metagpt_root() assert project_root.name == 'metagpt' def test_get_root_exception(self): with pytest.raises(Exception) as exc_info: self.change_etc_dir() - get_project_root() + get_metagpt_root() assert str(exc_info.value) == "Project root not found." diff --git a/tests/metagpt/utils/test_output_parser.py b/tests/metagpt/utils/test_output_parser.py index 4e362f9f7..99ab1f79e 100644 --- a/tests/metagpt/utils/test_output_parser.py +++ b/tests/metagpt/utils/test_output_parser.py @@ -218,7 +218,7 @@ def case(): } t_text1 = '''## Original Requirements: -The boss wants to create a web-based version of the game "Fly Bird". +The user wants to create a web-based version of the game "Fly Bird". ## Product Goals: diff --git a/tests/metagpt/utils/test_read_docx.py b/tests/metagpt/utils/test_read_docx.py index a7d0774a8..adf473ae7 100644 --- a/tests/metagpt/utils/test_read_docx.py +++ b/tests/metagpt/utils/test_read_docx.py @@ -6,12 +6,12 @@ @File : test_read_docx.py """ -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.utils.read_document import read_docx class TestReadDocx: def test_read_docx(self): - docx_sample = PROJECT_ROOT / "tests/data/docx_for_test.docx" + docx_sample = METAGPT_ROOT / "tests/data/docx_for_test.docx" docx = read_docx(docx_sample) assert len(docx) == 6 From 715a1d874aa1d690aa9f7c5b27d404e6d9c1b19a Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 27 Nov 2023 15:48:07 +0800 Subject: [PATCH 157/232] fix config --- config/config.yaml | 6 +++--- metagpt/actions/SummarizeCode.py | 14 ++++++-------- metagpt/actions/write_code.py | 2 +- requirements.txt | 3 ++- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index bed67083c..9acdbe8a1 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,9 +7,9 @@ ## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" -#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model -OPENAI_API_MODEL: "gpt-4" -MAX_TOKENS: 1500 +#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model +OPENAI_API_MODEL: "gpt-4-1106-preview" +MAX_TOKENS: 4096 RPM: 10 #### if Spark diff --git a/metagpt/actions/SummarizeCode.py b/metagpt/actions/SummarizeCode.py index 1015d3bfb..49a350b75 100644 --- a/metagpt/actions/SummarizeCode.py +++ b/metagpt/actions/SummarizeCode.py @@ -5,11 +5,10 @@ @File : SummarizeCode.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -23,10 +22,10 @@ ## Code Review All: 请你对历史所有文件进行阅读，分析每个文件是否都完整实现了用户需求，找到可能的bug，如函数未实现、调用错误、未引用等 -## Summary: 根据历史文件的实现情况进行总结 - ## Call flow: 根据实现的函数，使用mermaid绘制完整的调用链 +## Summary: 根据历史文件的实现情况进行总结 + ## TODOs: 这里写出需要修改的文件列表，我们会在之后进行修改 """ @@ -80,14 +79,13 @@ def __init__(self, name="SummaryCode", context: list[Message] = None, llm=None): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code_review_all(self, prompt): + async def summarize_code(self, prompt): code_rsp = await self._aask(prompt) return code_rsp async def run(self, context): format_example = FORMAT_EXAMPLE.format() prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) - logger.info(f'Code review all..') - rsp = await self.write_code_review_all(prompt) + logger.info("Code review all..") + rsp = await self.summarize_code(prompt) return rsp - \ No newline at end of file diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 176718dfc..1f6d16b3b 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,13 +5,13 @@ @Author : alexanderwu @File : write_code.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions import WriteDesign from metagpt.actions.action import Action from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE diff --git a/requirements.txt b/requirements.txt index f0169d7fa..f233e398f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,8 @@ channels==4.0.0 # docx==0.2.4 #faiss==1.5.3 faiss_cpu==1.7.4 -fire==0.4.0 +# fire==0.4.0 +typer # godot==0.1.1 # google_api_python_client==2.93.0 lancedb==0.1.16 From 22288a342dcbb029447e6d896148bf22da4e9da3 Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 27 Nov 2023 15:36:50 +0800 Subject: [PATCH 158/232] =?UTF-8?q?1.=20=E5=8A=A8=E4=BD=9C=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20=20=201.=20SummarizeCode=E5=8A=A8=E4=BD=9C=EF=BC=9A?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E5=9F=BA=E4=BA=8E=E4=BB=A3=E7=A0=81=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=80=BB=E7=BB=93=EF=BC=8C=E6=80=9D=E8=80=83bug?= =?UTF-8?q?=E3=80=81=E9=80=BB=E8=BE=91=E3=80=81todo=20=20=202.=20CodeRevie?= =?UTF-8?q?w=E5=8A=A8=E4=BD=9C=E4=BC=98=E5=8C=96=EF=BC=9A=E7=9B=AE?= =?UTF-8?q?=E5=89=8D=E5=BC=BA=E5=88=B6=E8=A6=81=E6=B1=82=E5=9B=9E=E7=AD=94?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E6=9C=89=E6=9B=B4=E9=AB=98=E7=9A=84?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E7=8E=87=E4=BA=86=20=20=20=20=201.=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86LGTM/LBTM=E7=9A=84=E5=9B=9E=E7=AD=94?= =?UTF-8?q?=EF=BC=8C=E5=9C=A8LGTM=E6=97=B6=E4=BC=9A=E5=8F=8A=E6=97=B6?= =?UTF-8?q?=E5=81=9C=E6=AD=A2=EF=BC=8C=E4=B8=8D=E9=87=8D=E5=86=99=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=20=20=20=20=202.=20=E7=9B=AE=E5=89=8D=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E4=BA=86=E8=AE=BE=E7=BD=AE=E4=B8=AD=E7=9A=84=E5=8F=82?= =?UTF-8?q?=E6=95=B0code=5Freview=5Fk=5Ftimes=EF=BC=8C=E4=B8=8Ereflexion?= =?UTF-8?q?=E7=B1=BB=E4=BC=BC=EF=BC=8C=E8=AE=BE=E7=BD=AE=E4=B8=BA2=20=20?= =?UTF-8?q?=20=20=203.=20=E4=BB=8D=E7=84=B6=E6=9C=89=E6=A6=82=E7=8E=87?= =?UTF-8?q?=E5=8F=91=E7=94=9F=E6=8C=87=E4=BB=A4=E4=B8=8D=E9=81=B5=E5=BE=AA?= =?UTF-8?q?=EF=BC=8C=E5=B0=A4=E5=85=B6=E6=98=AF=E4=BC=9A=E6=9C=89=E6=AF=94?= =?UTF-8?q?=E8=BE=83=E9=AB=98=E7=9A=84=E6=A6=82=E7=8E=87=E5=8F=91=E7=94=9F?= =?UTF-8?q?=E5=90=8C=E6=97=B6review=E5=A4=9A=E4=B8=AA=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=96=87=E4=BB=B6=EF=BC=8C=E8=BF=98=E6=B2=A1=E6=83=B3=E5=A5=BD?= =?UTF-8?q?=E6=80=8E=E4=B9=88=E8=A7=A3=E5=86=B3=20#FIXME=20=20=203.=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86env=E5=88=B0Action=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E4=B8=AD=EF=BC=8C=E7=8E=B0=E5=9C=A8=E5=8F=AF=E4=BB=A5=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E8=B0=83=E7=94=A8=E7=8E=AF=E5=A2=83=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E4=BA=86=20=20=204.=20WriteDesign=EF=BC=9A=E5=8E=BB=E9=99=A4?= =?UTF-8?q?=E4=BA=86=E5=AF=B9project=5Fname=E7=9A=84=E7=BA=A0=E6=AD=A3?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=8C=E7=8E=B0=E5=9C=A8=E5=BC=95=E5=AF=BC?= =?UTF-8?q?=E4=B8=8B=E5=8F=AF=E4=BB=A5=E4=B8=80=E6=AC=A1=E7=94=9F=E6=88=90?= =?UTF-8?q?=E5=AF=B9=20=20=20=20=201.=20=E4=BF=AE=E6=94=B9=E4=BA=86?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E8=AF=8D=E4=B8=AD=E7=9A=84##=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=EF=BC=8C=E6=94=B9=E4=B8=BA=E4=BA=86JSON=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=202.=20=E6=95=B0=E6=8D=AE=E7=BB=93=E6=9E=84=20=20=201?= =?UTF-8?q?.=20Document=E7=9A=84=E6=A0=87=E5=87=86=E5=8C=96=EF=BC=9AEnv->R?= =?UTF-8?q?epo->Document=EF=BC=8C=E5=85=B6=E4=B8=ADDocument/Asset/Code?= =?UTF-8?q?=E9=83=BD=E6=98=AFDocument=20=20=20=20=201.=20=E5=8E=9F?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E6=A3=80=E7=B4=A2=E7=9A=84Document=E6=94=B9?= =?UTF-8?q?=E4=B8=BAIndexableDocument=20=20=202.=20Repo=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=BC=95=E5=85=A5=EF=BC=9A=E7=94=A8=E4=BA=8EDocument=E8=A3=85?= =?UTF-8?q?=E8=BD=BD=E4=B8=8E=E5=85=83=E6=95=B0=E6=8D=AE=E8=A3=85=E8=BD=BD?= =?UTF-8?q?=20=20=203.=20RepoParser=E5=BC=95=E5=85=A5=EF=BC=9A=E5=86=99?= =?UTF-8?q?=E4=BA=86=E4=B8=80=E4=B8=AA=E7=AE=80=E5=8D=95=E7=9A=84AST=20par?= =?UTF-8?q?ser=EF=BC=88=E5=90=8E=E7=BB=AD=E5=8F=AF=E8=83=BD=E8=A6=81?= =?UTF-8?q?=E6=8D=A2tree-sitter=EF=BC=89=EF=BC=8C=E7=BB=99=E5=87=BA?= =?UTF-8?q?=E4=BA=86=E6=95=B4=E5=BA=93symbol=20=20=204.=20Env=E4=B8=AD?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86set/get/set=5Fdoc/get=5Fdoc=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E7=94=A8=E4=BA=8Eset/get=E5=8D=95=E4=B8=AA?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E6=88=96=E8=80=85=E4=B8=80=E4=B8=AADocument?= =?UTF-8?q?=E3=80=82=E8=BF=99=E4=B8=AA=E9=80=BB=E8=BE=91=E5=90=8E=E7=BB=AD?= =?UTF-8?q?=E6=88=96=E8=AE=B8=E4=BC=9A=E8=BF=9B=E4=B8=80=E6=AD=A5=E7=AE=80?= =?UTF-8?q?=E5=8C=96=203.=20=E9=85=8D=E7=BD=AE=E4=BC=98=E5=8C=96=20=20=201?= =?UTF-8?q?.=20=E9=BB=98=E8=AE=A4=E6=9B=B4=E6=8D=A2=E4=B8=BAgpt-4-1106-pre?= =?UTF-8?q?view=EF=BC=8C=E4=BB=A5=E8=8E=B7=E5=BE=97=E6=9C=80=E5=A5=BD?= =?UTF-8?q?=E7=9A=84=E6=95=88=E6=9E=9C=E4=B8=8E=E6=88=90=E6=9C=AC=20=20=20?= =?UTF-8?q?2.=20=E6=8F=90=E4=BE=9B~/.metagpt=E4=BD=9C=E4=B8=BA=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=9C=80=E9=AB=98=E4=BC=98=E5=85=88=E7=BA=A7=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=EF=BC=8C=E4=BB=8E=E4=B8=AD=E8=AF=BB=E5=8F=96config.ya?= =?UTF-8?q?ml=20=20=203.=20workspace=E5=8F=AF=E4=BB=A5=E7=81=B5=E6=B4=BB?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E4=BA=86=EF=BC=8C=E5=9C=A8config=E4=B8=AD?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=20=20=204.=20project=5Fname=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E7=94=B1=E5=91=BD=E4=BB=A4=E8=A1=8C=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E4=B8=94=E6=94=B9=E4=B8=BA=E7=94=B1ProductMa?= =?UTF-8?q?nager=E7=94=9F=E6=88=90=204.=20metagpt=E4=BD=9C=E4=B8=BA?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=91=BD=E4=BB=A4=E8=A1=8C=EF=BC=8C=E8=80=8C?= =?UTF-8?q?=E9=9D=9Epython=20startup.py=20metagpt=20--help?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit metagpt --project-name game_2048 "make a 2048 game" metagpt "make a 2048 game" metagpt --project-name game_2048 --inc "将2048改为4096" metagpt --project-name game_2048 --auto-inc "make a 2048 game" 1. 使用新的METAGPT_ROOT生成方式，而非寻找git，以便cli安装 2. 命令行由fire换为了typer，它会带来相对更好的体验 3. project_name可以灵活指定了，在metagpt命令行输入中配置 5. 其他 1. 现在支持多国语言了，中文已测试 2. BossRequirement -> UserRequirement 3. 大量错误文本的修正，增加了可读性 4. 中量提示词优化，稍微提升了一些准确率 5. 暂时屏蔽了LongtermMemory相关逻辑，这个逻辑底层调用了langchain的FAISS，会带来~5秒加载耗时 6. 修复了安装包中的部分描述错误 7. 去除了config中在openai_proxy设定时对base的重复修改，这个修改应该在openai初始化时发生 8. 修复了JSON在中文存储时的特定问题，ensure_ascii=False --- examples/debate.py | 2 +- metagpt/actions/action.py | 4 + metagpt/actions/design_api.py | 37 +++--- metagpt/actions/project_management.py | 36 +++--- .../{SummarizeCode.py => summarize_code.py} | 21 ++-- metagpt/actions/write_code.py | 3 +- metagpt/actions/write_code_review.py | 84 +++++++++---- metagpt/actions/write_prd.py | 88 ++++++------- metagpt/actions/write_test.py | 2 +- metagpt/config.py | 6 +- metagpt/document.py | 118 ++++++++++++------ metagpt/environment.py | 30 ++++- metagpt/memory/longterm_memory.py | 2 +- metagpt/provider/openai_api.py | 2 + metagpt/{document_store => }/repo_parser.py | 48 +++---- metagpt/roles/engineer.py | 26 ++-- metagpt/roles/qa_engineer.py | 4 +- metagpt/roles/role.py | 15 +++ metagpt/startup.py | 7 +- metagpt/team.py | 10 +- tests/metagpt/actions/mock.py | 2 +- tests/metagpt/roles/mock.py | 2 +- tests/metagpt/roles/test_ui.py | 2 +- ...st_software_company.py => test_startup.py} | 13 +- 24 files changed, 360 insertions(+), 204 deletions(-) rename metagpt/actions/{SummarizeCode.py => summarize_code.py} (62%) rename metagpt/{document_store => }/repo_parser.py (67%) rename tests/metagpt/{test_software_company.py => test_startup.py} (51%) diff --git a/examples/debate.py b/examples/debate.py index 0f5d1591b..e62a5aaa1 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -88,7 +88,7 @@ async def debate(idea: str, investment: float = 3.0, n_round: int = 5): team = Team() team.hire([Biden, Trump]) team.invest(investment) - team.start_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first + team.run_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first await team.run(n_round=n_round) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 790295d55..f8016b8a2 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -30,6 +30,10 @@ def __init__(self, name: str = "", context=None, llm: LLM = None): self.desc = "" self.content = "" self.instruct_content = None + self.env = None + + def set_env(self, env): + self.env = env def set_prefix(self, prefix, profile): """Set prefix for later usage""" diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index f58d49495..9e2bfc12c 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -14,7 +14,6 @@ from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -27,11 +26,12 @@ {format_example} ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirement: Fill in the following missing information based on the context, each section name is a key in json ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -48,7 +48,7 @@ [CONTENT] { "Implementation approach": "We will ...", - "Python package name": "snake_game", + "project_name": "snake_game", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram @@ -78,12 +78,13 @@ class Game{ {format_example} ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. +ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -99,7 +100,7 @@ class Game{ ## Implementation approach We will ... -## Python package name +## project_name ```python "snake_game" ``` @@ -138,7 +139,7 @@ class Game{ OUTPUT_MAPPING = { "Implementation approach": (str, ...), - "Python package name": (str, ...), + "project_name": (str, ...), "File list": (List[str], ...), "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), @@ -170,7 +171,7 @@ async def _save_prd(self, docs_path, resources_path, context): if context[-1].instruct_content: logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) + prd_file.write_text(context[-1].instruct_content.json(ensure_ascii=False), encoding='utf-8') async def _save_system_design(self, docs_path, resources_path, system_design): data_api_design = system_design.instruct_content.dict()[ @@ -183,14 +184,14 @@ async def _save_system_design(self, docs_path, resources_path, system_design): await mermaid_to_file(seq_flow, resources_path / "seq_flow") system_design_file = docs_path / "system_design.md" logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) + system_design_file.write_text(system_design.instruct_content.json(ensure_ascii=False), encoding='utf-8') async def _save(self, context, system_design): if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["Python package name"] + project_name = system_design.instruct_content.dict()["project_name"] else: - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = CONFIG.workspace_path / ws_name + project_name = CodeParser.parse_str(block="project_name", text=system_design) + workspace = CONFIG.workspace_path / project_name self.recreate_workspace(workspace) docs_path = workspace / "docs" resources_path = workspace / "resources" @@ -204,11 +205,11 @@ async def run(self, context, format=CONFIG.prompt_format): prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr - setattr( - system_design.instruct_content, - "Python package name", - system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), - ) + # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr + # setattr( + # system_design.instruct_content, + # "project_name", + # system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), + # ) await self._save(context, system_design) return system_design diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 467cb4d83..805226a25 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -11,7 +11,6 @@ from metagpt.config import CONFIG from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown templates = { "json": { @@ -23,19 +22,20 @@ {format_example} ----- Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. +ATTENTION: Output carefully referenced "Format example" in format. ## Required Python third-party packages: Provide Python list[str] in requirements.txt format ## Required Other language third-party packages: Provide Python list[str] in requirements.txt format -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. ## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. @@ -52,17 +52,17 @@ "Required Other language third-party packages": [ "No third-party ..." ], - "Full API spec": """ - openapi: 3.0.0 - ... - description: A JSON object ... - """, "Logic Analysis": [ - ["game.py","Contains..."] + ["game.py", "Contains..."] ], "Task list": [ "game.py" ], + "Full API spec": """ + openapi: 3.0.0 + ... + description: A JSON object ... + """, "Shared Knowledge": """ 'game.py' contains ... """, @@ -86,12 +86,12 @@ ## Required Other language third-party packages: Provided in requirements.txt format -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. ## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. @@ -126,14 +126,16 @@ ## Logic Analysis ```python [ - ["game.py", "Contains ..."], + ["index.js", "Contains ..."], + ["main.py", "Contains ..."], ] ``` ## Task list ```python [ - "game.py", + "index.js", + "main.py", ] ``` @@ -167,11 +169,11 @@ def __init__(self, name="CreateTasks", context=None, llm=None): def _save(self, context, rsp): if context[-1].instruct_content: - ws_name = context[-1].instruct_content.dict()["Python package name"] + ws_name = context[-1].instruct_content.dict()["project_name"] else: - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) + ws_name = CodeParser.parse_str(block="project_name", text=context[-1].content) file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" - file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) + file_path.write_text(rsp.instruct_content.json(ensure_ascii=False)) # Write requirements.txt requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" diff --git a/metagpt/actions/SummarizeCode.py b/metagpt/actions/summarize_code.py similarity index 62% rename from metagpt/actions/SummarizeCode.py rename to metagpt/actions/summarize_code.py index 49a350b75..a85d3cdeb 100644 --- a/metagpt/actions/SummarizeCode.py +++ b/metagpt/actions/summarize_code.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ @Author : alexanderwu -@File : SummarizeCode.py +@File : summarize_code.py """ from tenacity import retry, stop_after_attempt, wait_fixed @@ -13,6 +13,7 @@ PROMPT_TEMPLATE = """ NOTICE Role: You are a professional software engineer, and your main task is to review the code. +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ----- @@ -20,13 +21,13 @@ {context} ----- -## Code Review All: 请你对历史所有文件进行阅读，分析每个文件是否都完整实现了用户需求，找到可能的bug，如函数未实现、调用错误、未引用等 +## Code Review All: 请你对历史所有文件进行阅读，在文件中找到可能的bug，如函数未实现、调用错误、未引用等 -## Call flow: 根据实现的函数，使用mermaid绘制完整的调用链 +## Call flow: mermaid代码，根据实现的函数，使用mermaid绘制完整的调用链 ## Summary: 根据历史文件的实现情况进行总结 -## TODOs: 这里写出需要修改的文件列表，我们会在之后进行修改 +## TODOs: Python dict[str, str]，这里写出需要修改的文件列表与理由，我们会在之后进行修改 """ @@ -67,15 +68,15 @@ - ... ## TODOs -1. ... -2. ... -3. ... +{ + "a.py": "implement requirement xxx...", +} """ class SummarizeCode(Action): - def __init__(self, name="SummaryCode", context: list[Message] = None, llm=None): + def __init__(self, name="SummarizeCode", context: list[Message] = None, llm=None): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) @@ -84,8 +85,8 @@ async def summarize_code(self, prompt): return code_rsp async def run(self, context): - format_example = FORMAT_EXAMPLE.format() + format_example = FORMAT_EXAMPLE prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) - logger.info("Code review all..") + logger.info("Summarize code..") rsp = await self.summarize_code(prompt) return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 1f6d16b3b..2631ec138 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -16,6 +16,7 @@ PROMPT_TEMPLATE = """ NOTICE Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ----- @@ -60,7 +61,7 @@ def _save(self, context, filename, code): design = [i for i in context if i.cause_by == WriteDesign][0] - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) + ws_name = CodeParser.parse_str(block="project_name", text=design.content) ws_path = CONFIG.workspace_path / ws_name if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): ws_path = ws_path / ws_name diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index c6538bf7b..aebe3f4fa 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -6,56 +6,84 @@ @File : write_code_review.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed +from metagpt.config import CONFIG PROMPT_TEMPLATE = """ NOTICE Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code Review: Based on the following context and code, follow the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. -2. Are there any issues with the code logic? If so, how to solve it? -3. Does the existing code follow the "Data structures and interfaces"? -4. Is there a function in the code that is not fully implemented? If so, how to implement it? -5. Does the code have unnecessary or lack dependencies? If so, how to solve it? - -## Rewrite Code: rewrite {filename} based on "Code Review" with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. ------ # Context {context} -## Code: {filename} +## Code to be Reviewed: {filename} ``` {code} ``` + ----- +## Code Review: Based on the "Code to be Reviewed", provide key, clear, concise, and specific code modification suggestions, up to 5. +1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. +2. Is the code logic completely correct? If there are errors, please indicate how to correct them. +3. Does the existing code follow the "Data structures and interfaces"? +4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step. +5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported +6. Is the code implemented concisely enough? Are methods from other files being reused correctly? + +## Code Review Result: If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. +LGTM/LBTM + +## Rewrite Code: if it still has some bugs, rewrite {filename} based on "Code Review" with triple quotes, try to get LGTM. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. RETURN ALL CODE, NEVER OMIT ANYTHING. 以任何方式省略代码都是不允许的。 +``` +``` + ## Format example ------ {format_example} ------ """ FORMAT_EXAMPLE = """ - -## Code Review +----- +# EXAMPLE 1 +## Code Review: {filename} 1. No, we should add the logic of ... 2. ... 3. ... 4. ... 5. ... +6. ... + +## Code Review Result: {filename} +LBTM ## Rewrite Code: {filename} ```python ## {filename} ... ``` +----- +# EXAMPLE 2 +## Code Review: {filename} +1. Yes. +2. Yes. +3. Yes. +4. Yes. +5. Yes. +6. Yes. + +## Code Review Result: {filename} +LGTM + +## Rewrite Code: {filename} +pass +----- """ @@ -64,17 +92,27 @@ def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=No super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code(self, prompt): + async def write_code_review_and_rewrite(self, prompt): code_rsp = await self._aask(prompt) + result = CodeParser.parse_block("Code Review Result", code_rsp) + if "LGTM" in result: + return result, None code = CodeParser.parse_code(block="", text=code_rsp) - return code + return result, code async def run(self, context, code, filename): - format_example = FORMAT_EXAMPLE.format(filename=filename) - prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) - logger.info(f'Code review {filename}..') - code = await self.write_code(prompt) + iterative_code = code + k = CONFIG.code_review_k_times + for i in range(k): + format_example = FORMAT_EXAMPLE.format(filename=filename) + prompt = PROMPT_TEMPLATE.format(context=context, code=iterative_code, filename=filename, format_example=format_example) + logger.info(f'Code review and rewrite {filename}: {i+1}/{k} | {len(iterative_code)=}, {len(code)=}') + result, rewrited_code = await self.write_code_review_and_rewrite(prompt) + if "LBTM" in result: + iterative_code = rewrited_code + elif "LGTM" in result: + return iterative_code # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) - return code - \ No newline at end of file + # 如果rewrited_code是None（原code perfect），那么直接返回code + return iterative_code diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 584d31998..4780762ca 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -17,54 +17,50 @@ "json": { "PROMPT_TEMPLATE": """ # Context -## Original Requirements -{requirements} - -## Search Information -{search_information} - -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` +{{ + "Original Requirements": "{requirements}", + "Search Information": "" +}} ## Format example {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, each section name is a key in json - -## Original Requirements: Provide as Plain text, place the polished complete original requirements here - -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories - -## Competitive Analysis: Provided as Python list[str], up to 8 competitive product analyses - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. +ATTENTION: Output carefully referenced "Format example" in format. -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. +## YOU NEED TO FULFILL THE BELOW JSON DOC -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. +{{ + "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. + "Original Requirements": "", # str, place the polished complete original requirements here + "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Search Information": "", + "Requirements": "", + "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. + "User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories + "Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses + # Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + "Competitive Quadrant Chart": "quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + Campaign A: [0.3, 0.6] + Campaign B: [0.45, 0.23] + Campaign C: [0.57, 0.69] + Campaign D: [0.78, 0.34] + Campaign E: [0.40, 0.34] + Campaign F: [0.35, 0.78]", + "Requirement Analysis": "", # Provide as Plain text. + "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards + "UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. + "Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it. +}} output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -72,6 +68,7 @@ "FORMAT_EXAMPLE": """ [CONTENT] { + "Language": "", "Original Requirements": "", "Search Information": "", "Requirements": "", @@ -132,9 +129,12 @@ {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product +Language: Please use the same language as the user requirement to answer, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. +## Language: Provide as Plain text, use the same language as the user requirement. + ## Original Requirements: Provide as Plain text, place the polished complete original requirements here ## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -207,6 +207,7 @@ } OUTPUT_MAPPING = { + "Language": (str, ...), "Original Requirements": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), @@ -232,11 +233,14 @@ async def run(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) logger.info(sas.result) logger.info(rsp) + # logger.info(format) prompt_template, format_example = get_template(templates, format) + # logger.info(prompt_template) + # logger.info(format_example) prompt = prompt_template.format( requirements=requirements, search_information=info, format_example=format_example ) - logger.debug(prompt) + # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 2f4988c09..9988fda16 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -3,7 +3,7 @@ """ @Time : 2023/5/11 22:12 @Author : alexanderwu -@File : environment.py +@File : write_test.py """ from metagpt.actions.action import Action from metagpt.logs import logger diff --git a/metagpt/config.py b/metagpt/config.py index 1a9cdb4d2..d30a337e3 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -54,10 +54,7 @@ def __init__(self, yaml_file=default_yaml_file): (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") - openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy - if openai_proxy: - openai.proxy = openai_proxy - openai.api_base = self.openai_api_base + self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) @@ -87,6 +84,7 @@ def __init__(self, yaml_file=default_yaml_file): logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 + self.code_review_k_times = 2 self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") self.mmdc = self._get("MMDC", "mmdc") diff --git a/metagpt/document.py b/metagpt/document.py index 044210218..cf0821421 100644 --- a/metagpt/document.py +++ b/metagpt/document.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : document.py """ - +from enum import Enum from typing import Union, Optional from pathlib import Path from pydantic import BaseModel, Field @@ -18,7 +18,9 @@ from langchain.text_splitter import CharacterTextSplitter from tqdm import tqdm +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.repo_parser import RepoParser def validate_cols(content_col: str, df: pd.DataFrame): @@ -48,42 +50,56 @@ def read_data(data_path: Path): return data +class DocumentStatus(Enum): + """Indicates document status, a mechanism similar to RFC/PEP""" + DRAFT = "draft" + UNDERREVIEW = "underreview" + APPROVED = "approved" + DONE = "done" + + class Document(BaseModel): """ Document: Handles operations related to document files. """ - content: str = Field(default='') - file_path: Path = Field(default=None) + path: Path = Field(default=None) + name: str = Field(default="") + content: str = Field(default="") + + # metadata? in content perhaps. + author: str = Field(default="") + status: DocumentStatus = Field(default=DocumentStatus.DRAFT) + reviews: list = Field(default_factory=list) @classmethod - def from_path(cls, file_path: Path): + def from_path(cls, path: Path): """ Create a Document instance from a file path. """ - if not file_path.exists(): - raise FileNotFoundError(f"File {file_path} not found.") - content = file_path.read_text() - return cls(content=content, file_path=file_path) + if not path.exists(): + raise FileNotFoundError(f"File {path} not found.") + content = path.read_text() + return cls(content=content, path=path) @classmethod - def from_text(cls, text: str, file_path: Optional[Path] = None): + def from_text(cls, text: str, path: Optional[Path] = None): """ Create a Document from a text string. """ - return cls(content=text, file_path=file_path) + return cls(content=text, path=path) - def to_path(self, file_path: Optional[Path] = None): + def to_path(self, path: Optional[Path] = None): """ Save content to the specified file path. """ - if file_path is not None: - self.file_path = file_path + if path is not None: + self.path = path - if self.file_path is None: + if self.path is None: raise ValueError("File path is not set.") - self.file_path.parent.mkdir(parents=True, exist_ok=True) - self.file_path.write_text(self.content) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(self.content, encoding="utf-8") def persist(self): """ @@ -140,25 +156,35 @@ def get_docs_and_metadatas(self) -> (list, list): raise NotImplementedError("Data type not supported for metadata extraction.") +class RepoMetadata(BaseModel): + + name: str = Field(default="") + n_docs: int = Field(default=0) + n_chars: int = Field(default=0) + symbols: list = Field(default_factory=list) + + class Repo(BaseModel): # Name of this repo. name: str = Field(default="") + # metadata: RepoMetadata = Field(default=RepoMetadata) docs: dict[Path, Document] = Field(default_factory=dict) codes: dict[Path, Document] = Field(default_factory=dict) assets: dict[Path, Document] = Field(default_factory=dict) - repo_path: Path = Field(default_factory=Path) + path: Path = Field(default=None) def _path(self, filename): - return self.repo_path / filename + return self.path / filename @classmethod - def from_path(cls, repo_path: Path): + def from_path(cls, path: Path): """Load documents, code, and assets from a repository path.""" - repo_path.mkdir(parents=True, exist_ok=True) - repo = Repo(repo_path = repo_path) - for file_path in repo_path.rglob('*'): - if file_path.is_file(): + path.mkdir(parents=True, exist_ok=True) + repo = Repo(path=path, name=path.name) + for file_path in path.rglob('*'): + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if file_path.is_file() and file_path.suffix in [".json", ".txt", ".md", ".py", ".js", ".css", ".html"]: repo._set(file_path.read_text(), file_path) return repo @@ -171,23 +197,24 @@ def to_path(self): for asset in self.assets.values(): asset.to_path() - def _set(self, content: str, file_path: Path): + def _set(self, content: str, path: Path): """Add a document to the appropriate category based on its file extension.""" - file_ext = file_path.suffix - - doc = Document(content=content, file_path=file_path) - if file_ext.lower() == '.md': - self.docs[file_path] = doc - elif file_ext.lower() in ['.py', '.js', '.css', '.html']: - self.codes[file_path] = doc + suffix = path.suffix + doc = Document(content=content, path=path, name=str(path.relative_to(self.path))) + + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if suffix.lower() == '.md': + self.docs[path] = doc + elif suffix.lower() in ['.py', '.js', '.css', '.html']: + self.codes[path] = doc else: - self.assets[file_path] = doc + self.assets[path] = doc return doc def set(self, content: str, filename: str): """Set a document and persist it to disk.""" - file_path = self._path(filename) - doc = self._set(content, file_path) + path = self._path(filename) + doc = self._set(content, path) doc.to_path() def get(self, filename: str) -> Optional[Document]: @@ -195,13 +222,32 @@ def get(self, filename: str) -> Optional[Document]: path = self._path(filename) return self.docs.get(path) or self.codes.get(path) or self.assets.get(path) + def get_text_documents(self) -> list[Document]: + return list(self.docs.values()) + list(self.codes.values()) -def main(): - repo1 = Repo.from_path(Path("/Users/alexanderwu/workspace/t1")) + def eda(self) -> RepoMetadata: + n_docs = sum(len(i) for i in [self.docs, self.codes, self.assets]) + n_chars = sum(sum(len(j.content) for j in i.values()) for i in [self.docs, self.codes, self.assets]) + symbols = RepoParser(base_directory=self.path).generate_symbols() + return RepoMetadata(name=self.name, n_docs=n_docs, n_chars=n_chars, symbols=symbols) + + +def set_existing_repo(path=CONFIG.workspace_path / "t1"): + repo1 = Repo.from_path(path) repo1.set("wtf content", "doc/wtf_file.md") repo1.set("wtf code", "code/wtf_file.py") logger.info(repo1) # check doc +def load_existing_repo(path=CONFIG.workspace_path / "web_tetris"): + repo = Repo.from_path(path) + logger.info(repo) + logger.info(repo.eda()) + + +def main(): + load_existing_repo() + + if __name__ == '__main__': main() diff --git a/metagpt/environment.py b/metagpt/environment.py index 38077c90d..44c9b1c67 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -7,10 +7,12 @@ """ import asyncio from typing import Iterable +from pathlib import Path from pydantic import BaseModel, Field # from metagpt.document import Document +from metagpt.logs import logger from metagpt.document import Repo from metagpt.memory import Memory from metagpt.roles import Role @@ -26,6 +28,7 @@ class Environment(BaseModel): memory: Memory = Field(default_factory=Memory) history: str = Field(default='') repo: Repo = Field(default_factory=Repo) + kv: dict = Field(default_factory=dict) class Config: arbitrary_types_allowed = True @@ -52,9 +55,32 @@ def publish_message(self, message: Message): self.memory.add(message) self.history += f"\n{message}" - def publish_doc(self, content: str, filename: str): + def set_doc(self, content: str, filename: str): """向当前环境发布文档（包括代码）""" - self.repo.set(content, filename) + return self.repo.set(content, filename) + + def get_doc(self, filename: str): + return self.repo.get(filename) + + def set(self, k: str, v: str): + self.kv[k] = v + + def get(self, k: str): + return self.kv.get(k, None) + + def load_existing_repo(self, path: Path, inc: bool): + self.repo = Repo.from_path(path) + logger.info(self.repo.eda()) + + # Incremental mode: publish all docs to messages. Then roles can read the docs. + if inc: + docs = self.repo.get_text_documents() + for doc in docs: + msg = Message(content=doc.content) + self.publish_message(msg) + logger.info(f"Message from existing doc {doc.path}: {msg}") + logger.info(f"Load {len(docs)} docs from existing repo.") + raise NotImplementedError async def run(self, k=1): """处理一次所有信息的运行 diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index f8abea5f3..b21f80b7d 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -28,7 +28,7 @@ def recover_memory(self, role_id: str, rc: "RoleContext"): logger.warning(f"It may the first time to run Agent {role_id}, the long-term memory is empty") else: logger.warning( - f"Agent {role_id} has existed memory storage with {len(messages)} messages " f"and has recovered them." + f"Agent {role_id} has existing memory storage with {len(messages)} messages " f"and has recovered them." ) self.msg_from_recover = True self.add_batch(messages) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 34e5693f8..8ac0c4b21 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -157,6 +157,8 @@ def __init_openai(self, config): if config.openai_api_type: openai.api_type = config.openai_api_type openai.api_version = config.openai_api_version + if config.openai_proxy: + openai.proxy = config.openai_proxy self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: diff --git a/metagpt/document_store/repo_parser.py b/metagpt/repo_parser.py similarity index 67% rename from metagpt/document_store/repo_parser.py rename to metagpt/repo_parser.py index f7e2b0f4a..0020d47aa 100644 --- a/metagpt/document_store/repo_parser.py +++ b/metagpt/repo_parser.py @@ -6,15 +6,19 @@ @File : repo_parser.py """ import json -import pathlib -import ast +from pathlib import Path +import ast import pandas as pd +from pydantic import BaseModel, Field +from pprint import pformat + +from metagpt.config import CONFIG +from metagpt.logs import logger -class RepoParser: - def __init__(self): - self.base_directory = None +class RepoParser(BaseModel): + base_directory: Path = Field(default=None) def parse_file(self, file_path): """Parse a Python file in the repository.""" @@ -38,43 +42,42 @@ def extract_class_and_function_info(self, tree, file_path): file_info["classes"].append({"name": node.name, "methods": class_methods}) elif is_func(node): file_info["functions"].append(node.name) - elif isinstance(node, ast.Assign) or isinstance(node, ast.AnnAssign): + elif isinstance(node, (ast.Assign, ast.AnnAssign)): for target in node.targets if isinstance(node, ast.Assign) else [node.target]: if isinstance(target, ast.Name): file_info["globals"].append(target.id) return file_info - def generate_json_structure(self, directory, output_path): - """Generate a JSON file documenting the repository structure.""" + def generate_symbols(self): files_classes = [] + directory = self.base_directory for path in directory.rglob('*.py'): tree = self.parse_file(path) file_info = self.extract_class_and_function_info(tree, path) files_classes.append(file_info) + return files_classes + + def generate_json_structure(self, output_path): + """Generate a JSON file documenting the repository structure.""" + files_classes = self.generate_symbols() output_path.write_text(json.dumps(files_classes, indent=4)) - def generate_dataframe_structure(self, directory, output_path): + def generate_dataframe_structure(self, output_path): """Generate a DataFrame documenting the repository structure and save as CSV.""" - files_classes = [] - for path in directory.rglob('*.py'): - tree = self.parse_file(path) - file_info = self.extract_class_and_function_info(tree, path) - files_classes.append(file_info) - + files_classes = self.generate_symbols() df = pd.DataFrame(files_classes) df.to_csv(output_path, index=False) - def generate_structure(self, directory_path, output_path=None, mode='json'): + def generate_structure(self, output_path=None, mode='json'): """Generate the structure of the repository as a specified format.""" - self.base_directory = pathlib.Path(directory_path) output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}" - output_path = pathlib.Path(output_path) if output_path else output_file + output_path = Path(output_path) if output_path else output_file if mode == 'json': - self.generate_json_structure(self.base_directory, output_path) + self.generate_json_structure(output_path) elif mode == 'csv': - self.generate_dataframe_structure(self.base_directory, output_path) + self.generate_dataframe_structure(output_path) def is_func(node): @@ -82,8 +85,9 @@ def is_func(node): def main(): - repo_parser = RepoParser() - repo_parser.generate_structure("/Users/alexanderwu/git/mg1/metagpt", "/Users/alexanderwu/git/mg1/mg1-structure.csv", mode='csv') + repo_parser = RepoParser(base_directory=CONFIG.workspace_path / "web_2048") + symbols = repo_parser.generate_symbols() + logger.info(pformat(symbols)) if __name__ == '__main__': diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 171af47f0..e3f36b50d 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,7 +11,7 @@ from pathlib import Path from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.actions.SummarizeCode import SummarizeCode +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role @@ -74,8 +74,8 @@ def __init__( super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review - if self.use_code_review: - self._init_actions([WriteCode, WriteCodeReview]) + # if self.use_code_review: + # self._init_actions([WriteCode, WriteCodeReview]) self._watch([WriteTasks]) self.todos = [] self.n_borg = n_borg @@ -93,8 +93,8 @@ def parse_code(cls, code_text: str) -> str: @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + return system_design_msg.instruct_content.dict().get("project_name").strip().strip("'").strip('"') + return CodeParser.parse_str(block="project_name", text=system_design_msg.content) def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] @@ -182,16 +182,16 @@ async def _act_sp_with_cr(self) -> Message: msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) - context_str = "\n".join(context) + context_str = "\n----------\n".join(context) # Write code code = await WriteCode().run(context=context_str, filename=todo) # Code review if self.use_code_review: - try: - rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) + # try: + rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) + code = rewrite_code + # except Exception as e: + # logger.error("code review failed!", e) file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -203,8 +203,8 @@ async def _act_sp_with_cr(self) -> Message: msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) - context_str = "\n".join(context) - code_review_all = await SummarizeCode().run(context=context_str) + context_str = "\n----------\n".join(context) + summary = await SummarizeCode().run(context=context_str) logger.info(f"Done {self.get_workspace()} generating.") msg = Message( diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index f124646b3..313fe4aba 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -45,8 +45,8 @@ def __init__( @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + return system_design_msg.instruct_content.dict().get("project_name") + return CodeParser.parse_str(block="project_name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index d772c0748..5c5e7b76d 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -50,6 +50,7 @@ {name}: {result} """ + class RoleReactMode(str, Enum): REACT = "react" BY_ORDER = "by_order" @@ -59,6 +60,7 @@ class RoleReactMode(str, Enum): def values(cls): return [item.value for item in cls] + class RoleSetting(BaseModel): """Role Settings""" name: str @@ -131,6 +133,7 @@ def _init_actions(self, actions): logger.warning(f"is_human attribute does not take effect," f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action + i.set_env(self._rc.env) i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) self._states.append(f"{idx}. {action}") @@ -172,6 +175,18 @@ def set_env(self, env: 'Environment'): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env + def set_doc(self, content: str, filename: str): + return self._rc.env.set_doc(content, filename) + + def get_doc(self, filename: str): + return self._rc.env.get_doc(filename) + + def set(self, k, v): + return self._rc.env.set(k, v) + + def get(self, k): + return self._rc.env.get(k) + @property def profile(self): """Get the role description (position)""" diff --git a/metagpt/startup.py b/metagpt/startup.py index d8ca4072f..38f457fc2 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -15,7 +15,8 @@ def startup( code_review: bool = typer.Option(True, help="Whether to use code review."), run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), implement: bool = typer.Option(True, help="Enable or disable code implementation."), - project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'"), + project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."), + inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."), ): """Run a startup. Be a boss.""" from metagpt.roles import ProductManager, Architect, ProjectManager, Engineer, QaEngineer @@ -37,9 +38,9 @@ def startup( company.hire([QaEngineer()]) company.invest(investment) - company.start_project(project_name, idea) + company.run_project(idea, project_name=project_name, inc=inc) asyncio.run(company.run(n_round=n_round)) if __name__ == "__main__": - app() + startup(idea="Make a 2048 game.") diff --git a/metagpt/team.py b/metagpt/team.py index 2332aaa46..a22a09fe4 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -42,15 +42,19 @@ def _check_balance(self): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, project_name, idea, send_to: str = ""): + def run_project(self, idea, send_to: str = "", project_name: str = "", inc: bool = False): """Start a project from publishing user requirement.""" self.idea = idea # If user set project_name, then use it. - self.env.repo.name = project_name + if project_name: + path = CONFIG.workspace_path / project_name + self.env.load_existing_repo(path, inc=inc) + + # Human requirement. self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) def _save(self): - logger.info(self.json()) + logger.info(self.json(ensure_ascii=False)) async def run(self, n_round=3): """Run company until target round or no money""" diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index 5be1d8001..d367e253e 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -90,7 +90,7 @@ For testing, we can use the PyTest framework. This is a mature full-featured Python testing tool that helps you write better programs. -## Python package name: +## project_name: ```python "adventure_game" ``` diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index fbad06acb..c06844389 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -71,7 +71,7 @@ ``` ''' -SYSTEM_DESIGN = '''## Python package name +SYSTEM_DESIGN = '''## project_name ```python "smart_search_engine" ``` diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index d58d31bd9..ec507f75d 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -18,5 +18,5 @@ async def test_ui_role(idea: str, investment: float = 3.0, n_round: int = 5): company = Team() company.hire([ProductManager(), UI()]) company.invest(investment) - company.start_project(idea) + company.run_project(idea) await company.run(n_round=n_round) diff --git a/tests/metagpt/test_software_company.py b/tests/metagpt/test_startup.py similarity index 51% rename from tests/metagpt/test_software_company.py rename to tests/metagpt/test_startup.py index 4fc651f52..53a8d8735 100644 --- a/tests/metagpt/test_software_company.py +++ b/tests/metagpt/test_startup.py @@ -3,17 +3,26 @@ """ @Time : 2023/5/15 11:40 @Author : alexanderwu -@File : test_software_company.py +@File : test_startup.py """ import pytest +from typer.testing import CliRunner + +runner = CliRunner() from metagpt.logs import logger from metagpt.team import Team +from metagpt.startup import app @pytest.mark.asyncio async def test_team(): company = Team() - company.start_project("做一个基础搜索引擎，可以支持知识库") + company.run_project("做一个基础搜索引擎，可以支持知识库") history = await company.run(n_round=5) logger.info(history) + + +# def test_startup(): +# args = ["Make a 2048 game"] +# result = runner.invoke(app, args) From 70a0be3300a3b7e3d40a7ae55692f36ab8e46ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:49:05 +0800 Subject: [PATCH 159/232] feat: +annotation --- metagpt/const.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/const.py b/metagpt/const.py index 49965b622..a8c7356ca 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,8 +4,9 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, added key definitions for +@Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. +@Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. """ from pathlib import Path From ad6cf62d2113bb16b9ce93656d353dcbe06a2657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:57:19 +0800 Subject: [PATCH 160/232] feat: +annotation --- metagpt/roles/engineer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index b6ecc4767..3cf1f2125 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -10,6 +10,9 @@ 2. Consolidate message reception and processing logic within `_observe`. 3. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. 4. Supplemented the external transmission of internal messages. +@Modified By: mashenquan, 2023-11-27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. """ from __future__ import annotations @@ -97,11 +100,11 @@ async def _act_sp_precision(self, review=False) -> Set[str]: async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" changed_files = await self._act_sp_precision(review=self.use_code_review) - # 仅单测 + # Unit tests only. if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: changed_files.add(CONFIG.REQA_FILENAME) - from metagpt.roles import QaEngineer # 避免循环引用 + from metagpt.roles import QaEngineer # Avoid circular references. msg = Message( content="\n".join(changed_files), @@ -122,7 +125,7 @@ async def _think(self) -> Action | None: design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_files = Documents() - # 由上游变化导致的recode + # Recode caused by upstream changes. for filename in changed_task_files: design_doc = await design_file_repo.get(filename) task_doc = await task_file_repo.get(filename) @@ -144,7 +147,7 @@ async def _think(self) -> Action | None: ) changed_files.docs[task_filename] = coding_doc self.todos = [WriteCode(context=i, llm=self._llm) for i in changed_files.docs.values()] - # 用户直接修改的code + # Code directly modified by the user. dependency = await CONFIG.git_repo.get_dependency() for filename in changed_src_files: if filename in changed_files.docs: From 94ab03d2daf66bc81db70a8a7f30323555f86ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:02:18 +0800 Subject: [PATCH 161/232] feat: +annotation --- metagpt/roles/product_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 81577ec2c..bc6771829 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : product_manager.py +@Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135. """ from metagpt.actions import BossRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments From 41549e628082156e1bd613aa337665e95c2d685c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:11:49 +0800 Subject: [PATCH 162/232] feat: +annotation --- metagpt/roles/qa_engineer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a88b01e37..763ab6a3f 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -6,6 +6,11 @@ @File : qa_engineer.py @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. +@Modified By: mashenquan, 2023-11-27. + 1. Following the think-act principle, solidify the task parameters when creating the + WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. + 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message + to using file references. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG From e656e55f304f79a05591bad33dd88df6230b0d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:15:55 +0800 Subject: [PATCH 163/232] feat: +annotation --- metagpt/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index e910fc866..959e70dc1 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -6,7 +6,10 @@ @File : schema.py @Modified By: mashenquan, 2023-10-31. According to Chapter 2.2.1 of RFC 116: Replanned the distribution of responsibilities and functional positioning of `Message` class attributes. -@Modified By: mashenquan, 2023/11/22. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/11/22. + 1. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. + 2. Encapsulate the common key-values set to pydantic structures to standardize and unify parameter passing + between actions. """ from __future__ import annotations From 4c296a348b0f59b5889f9baf207ff0b91619b982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:18:39 +0800 Subject: [PATCH 164/232] feat: +annotation --- metagpt/software_company.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 5aa0864e0..72f28ab1d 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,6 +4,8 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023/11/27. Add an archiving operation after completing the project, as specified in + Section 2.2.3.3 of RFC 135. """ from pydantic import BaseModel, Field From 0c84c2c212bf6892ecbd9d4fb28c2135a881379c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:21:53 +0800 Subject: [PATCH 165/232] feat: +annotation --- metagpt/utils/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 9002a8dfb..fd3958a61 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -6,6 +6,8 @@ @File : common.py @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116: Add generic class-to-string and object-to-string conversion functionality. +@Modified By: mashenquan, 2023/11/27. Bug fix: `parse_recipient` failed to parse the recipient in certain GPT-3.5 + responses. """ import ast import contextlib From bd5daeb4e6743dd859d1d341f39eb95efcfa1b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:27:35 +0800 Subject: [PATCH 166/232] feat: +annotation --- metagpt/utils/dependency_file.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index 429027c7a..653e07ef9 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -18,11 +18,21 @@ class DependencyFile: + """A class representing a DependencyFile for managing dependencies. + + :param workdir: The working directory path for the DependencyFile. + """ + def __init__(self, workdir: Path | str): + """Initialize a DependencyFile instance. + + :param workdir: The working directory path for the DependencyFile. + """ self._dependencies = {} self._filename = Path(workdir) / ".dependencies.json" async def load(self): + """Load dependencies from the file asynchronously.""" if not self._filename.exists(): return try: @@ -33,6 +43,7 @@ async def load(self): logger.error(f"Failed to load {str(self._filename)}, error:{e}") async def save(self): + """Save dependencies to the file asynchronously.""" try: data = json.dumps(self._dependencies) async with aiofiles.open(str(self._filename), mode="w") as writer: @@ -41,6 +52,12 @@ async def save(self): logger.error(f"Failed to save {str(self._filename)}, error:{e}") async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): + """Update dependencies for a file asynchronously. + + :param filename: The filename or path. + :param dependencies: The set of dependencies. + :param persist: Whether to persist the changes immediately. + """ if persist: await self.load() @@ -65,6 +82,12 @@ async def update(self, filename: Path | str, dependencies: Set[Path | str], pers await self.save() async def get(self, filename: Path | str, persist=False): + """Get dependencies for a file asynchronously. + + :param filename: The filename or path. + :param persist: Whether to load dependencies from the file immediately. + :return: A set of dependencies. + """ if persist: await self.load() @@ -76,8 +99,10 @@ async def get(self, filename: Path | str, persist=False): return set(self._dependencies.get(str(key), {})) def delete_file(self): + """Delete the dependency file.""" self._filename.unlink(missing_ok=True) @property def exists(self): + """Check if the dependency file exists.""" return self._filename.exists() From 66fc1b83509cf0d85556a339c21941232ad2934c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 17:43:20 +0800 Subject: [PATCH 167/232] feat: merge geekan:main --- ...mit-config.yaml => .pre-commit-config.yam_ | 0 README.md | 297 ++++-------------- config/config.yaml | 5 +- docs/README_CN.md | 213 ++++--------- docs/README_JA.md | 33 +- examples/build_customized_agent.py | 60 +--- examples/debate.py | 99 ++---- metagpt/config.py | 9 +- metagpt/const.py | 9 +- metagpt/llm.py | 29 +- metagpt/provider/base_chatbot.py | 1 + metagpt/provider/base_gpt_api.py | 55 +++- metagpt/provider/openai_api.py | 84 ++++- metagpt/roles/engineer.py | 1 + metagpt/roles/invoice_ocr_assistant.py | 26 +- metagpt/roles/researcher.py | 19 +- metagpt/roles/role.py | 126 ++++++-- metagpt/software_company.py | 59 +--- metagpt/utils/mermaid.py | 5 +- metagpt/utils/token_counter.py | 6 +- requirements.txt | 4 +- setup.py | 6 +- startup.py | 4 +- tests/metagpt/roles/test_ui.py | 4 +- tests/metagpt/test_software_company.py | 6 +- 25 files changed, 480 insertions(+), 680 deletions(-) rename .pre-commit-config.yaml => .pre-commit-config.yam_ (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yam_ similarity index 100% rename from .pre-commit-config.yaml rename to .pre-commit-config.yam_ diff --git a/README.md b/README.md index 70460ceb4..e80082a3a 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,13 @@ CN doc EN doc JA doc -Discord Follow +Discord Follow License: MIT roadmap Twitter Follow

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face @@ -33,132 +32,38 @@

Software Company Multi-Role Schematic (Gradually Implementing)

-## MetaGPT's Abilities -https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 - - - -## Examples (fully generated by GPT-4) - -For example, if you type `python startup.py "Design a RecSys like Toutiao"`, you would get many outputs, one of them is data & api design - -![Jinri Toutiao Recsys Data & API Design](docs/resources/workspace/content_rec_sys/resources/data_api_design.png) +## Install -It costs approximately **$0.2** (in GPT-4 API fees) to generate one example with analysis and design, and around **$2.0** for a full project. +### Pip installation +```bash +# Step 1: Ensure that Python 3.9+ is installed on your system. You can check this by using: +# You can use conda to initialize a new python env +# conda create -n metagpt python=3.9 +# conda activate metagpt +python3 --version +# Step 2: Clone the repository to your local machine for latest version, and install it. +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT +pip3 install -e. # or pip3 install metagpt # for stable version +# Step 3: run the startup.py +# setup your OPENAI_API_KEY in key.yaml copy from config.yaml +python3 startup.py "Write a cli snake game" -## Installation - -### Installation Video Guide - -- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) - -### Traditional Installation - -```bash -# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) +# Step 4 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step. +# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) npm --version sudo npm install -g @mermaid-js/mermaid-cli - -# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: -python --version - -# Step 3: Clone the repository to your local machine, and install it. -git clone https://github.com/geekan/metagpt -cd metagpt -pip install -e. ``` -**Note:** - -- If already have Chrome, Chromium, or MS Edge installed, you can skip downloading Chromium by setting the environment variable - `PUPPETEER_SKIP_CHROMIUM_DOWNLOAD` to `true`. - -- Some people are [having issues](https://github.com/mermaidjs/mermaid.cli/issues/15) installing this tool globally. Installing it locally is an alternative solution, - - ```bash - npm install @mermaid-js/mermaid-cli - ``` - -- don't forget to the configuration for mmdc in config.yml - - ```yml - PUPPETEER_CONFIG: "./config/puppeteer-config.json" - MMDC: "./node_modules/.bin/mmdc" - ``` - -- if `pip install -e.` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `pip install -e. --user` - -- To convert Mermaid charts to SVG, PNG, and PDF formats. In addition to the Node.js version of Mermaid-CLI, you now have the option to use Python version Playwright, pyppeteer or mermaid.ink for this task. +detail installation please refer to [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) - - Playwright - - **Install Playwright** - - ```bash - pip install playwright - ``` - - - **Install the Required Browsers** - - to support PDF conversion, please install Chrominum. - - ```bash - playwright install --with-deps chromium - ``` - - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `playwright` - - ```yaml - MERMAID_ENGINE: playwright - ``` - - - pyppeteer - - **Install pyppeteer** - - ```bash - pip install pyppeteer - ``` - - - **Use your own Browsers** - - pyppeteer allows you use installed browsers, please set the following envirment - - ```bash - export PUPPETEER_EXECUTABLE_PATH = /path/to/your/chromium or edge or chrome - ``` - - please do not use this command to install browser, it is too old - - ```bash - pyppeteer-install - ``` - - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `pyppeteer` - - ```yaml - MERMAID_ENGINE: pyppeteer - ``` - - - mermaid.ink - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `ink` - - ```yaml - MERMAID_ENGINE: ink - ``` - - Note: this method does not support pdf export. - -### Installation by Docker +### Docker installation +> Note: In the Windows, you need to replace "/opt/metagpt" with a directory that Docker has permission to create, such as "D:\Users\x\metagpt" ```bash # Step 1: Download metagpt official image and prepare config.yaml @@ -174,156 +79,60 @@ docker run --rm \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ python startup.py "Write a cli snake game" - -# You can also start a container and execute commands in it -docker run --name metagpt -d \ - --privileged \ - -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ - -v /opt/metagpt/workspace:/app/metagpt/workspace \ - metagpt/metagpt:latest - -docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" -``` - -The command `docker run ...` do the following things: - -- Run in privileged mode to have permission to run the browser -- Map host configure file `/opt/metagpt/config/key.yaml` to container `/app/metagpt/config/key.yaml` -- Map host directory `/opt/metagpt/workspace` to container `/app/metagpt/workspace` -- Execute the demo command `python startup.py "Write a cli snake game"` - -### Build image by yourself - -```bash -# You can also build metagpt image by yourself. -git clone https://github.com/geekan/MetaGPT.git -cd MetaGPT && docker build -t metagpt:custom . ``` -## Configuration - -- Configure your `OPENAI_API_KEY` in any of `config/key.yaml / config/config.yaml / env` -- Priority order: `config/key.yaml > config/config.yaml > env` +detail installation please refer to [docker_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-with-docker) -```bash -# Copy the configuration file and make the necessary modifications. -cp config/config.yaml config/key.yaml -``` - -| Variable Name | config/key.yaml | env | -| ------------------------------------------ | ----------------------------------------- | ----------------------------------------------- | -| OPENAI_API_KEY # Replace with your own key | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # Optional | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | - -## Tutorial: Initiating a startup - -```shell -# Run the script -python startup.py "Write a cli snake game" -# Do not hire an engineer to implement the project -python startup.py "Write a cli snake game" --implement False -# Hire an engineer and perform code reviews -python startup.py "Write a cli snake game" --code_review True -``` - -After running the script, you can find your new project in the `workspace/` directory. - -### Preference of Platform or Tool - -You can tell which platform or tool you want to use when stating your requirements. - -```shell -python startup.py "Write a cli snake game based on pygame" -``` +### QuickStart & Demo Video +- Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) +- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) +- [Official Demo Video](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) -### Usage +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 -``` -NAME - startup.py - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - -SYNOPSIS - startup.py IDEA - -DESCRIPTION - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - -POSITIONAL ARGUMENTS - IDEA - Type: str - Your innovative idea, such as "Creating a snake game." - -FLAGS - --investment=INVESTMENT - Type: float - Default: 3.0 - As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - --n_round=N_ROUND - Type: int - Default: 5 - -NOTES - You can also use flags syntax for POSITIONAL ARGUMENTS -``` +## Tutorial -### Code walkthrough +- 🗒 [Online Document](https://docs.deepwisdom.ai/) +- 💻 [Usage](https://docs.deepwisdom.ai/guide/get_started/quickstart.html) +- 🔎 [What can MetaGPT do?](https://docs.deepwisdom.ai/guide/get_started/introduction.html) +- 🛠 How to build your own agents? + - [MetaGPT Usage & Development Guide | Agent 101](https://docs.deepwisdom.ai/guide/tutorials/agent_101.html) + - [MetaGPT Usage & Development Guide | MultiAgent 101](https://docs.deepwisdom.ai/guide/tutorials/multi_agent_101.html) +- 🧑‍💻 Contribution + - [Develop Roadmap](docs/ROADMAP.md) +- 🔖 Use Cases + - [Debate](https://docs.deepwisdom.ai/guide/use_cases/multi_agent/debate.html) + - [Researcher](https://docs.deepwisdom.ai/guide/use_cases/agent/researcher.html) + - [Recepit Assistant](https://docs.deepwisdom.ai/guide/use_cases/agent/receipt_assistant.html) +- ❓ [FAQs](https://docs.deepwisdom.ai/guide/faq.html) -```python -from metagpt.software_company import SoftwareCompany -from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer +## Support -async def startup(idea: str, investment: float = 3.0, n_round: int = 5): - """Run a startup. Be a boss.""" - company = SoftwareCompany() - company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) -``` +### Discard Join US +📢 Join Our [Discord Channel](https://discord.gg/ZRHeExS6xv)! -You can check `examples` for more details on single role (with knowledge base) and LLM only examples. +Looking forward to seeing you there! 🎉 -## QuickStart +### Contact Information -It is difficult to install and configure the local environment for some users. The following tutorials will allow you to quickly experience the charm of MetaGPT. +If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions! -- [MetaGPT quickstart](https://deepwisdom.feishu.cn/wiki/CyY9wdJc4iNqArku3Lncl4v8n2b) +- **Email:** alexanderwu@fuzhi.ai +- **GitHub Issues:** For more technical inquiries, you can also create a new issue in our [GitHub repository](https://github.com/geekan/metagpt/issues). -Try it on Huggingface Space -- https://huggingface.co/spaces/deepwisdom/MetaGPT +We will respond to all questions within 2-3 business days. ## Citation -For now, cite the [Arxiv paper](https://arxiv.org/abs/2308.00352): +For now, cite the [arXiv paper](https://arxiv.org/abs/2308.00352): ```bibtex @misc{hong2023metagpt, - title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, - author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + title={MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework}, + author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Ceyao Zhang and Jinlin Wang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and Jürgen Schmidhuber}, year={2023}, eprint={2308.00352}, archivePrefix={arXiv}, primaryClass={cs.AI} } ``` - -## Contact Information - -If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions! - -- **Email:** alexanderwu@fuzhi.ai -- **GitHub Issues:** For more technical inquiries, you can also create a new issue in our [GitHub repository](https://github.com/geekan/metagpt/issues). - -We will respond to all questions within 2-3 business days. - -## Demo - -https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d - -## Join us - -📢 Join Our Discord Channel! -https://discord.gg/ZRHeExS6xv - -Looking forward to seeing you there! 🎉 diff --git a/config/config.yaml b/config/config.yaml index b2c50991d..bed67083c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,7 +7,7 @@ ## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" -#OPENAI_API_KEY: "YOUR_API_KEY" +#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model OPENAI_API_MODEL: "gpt-4" MAX_TOKENS: 1500 RPM: 10 @@ -31,6 +31,9 @@ RPM: 10 #DEPLOYMENT_NAME: "YOUR_DEPLOYMENT_NAME" #DEPLOYMENT_ID: "YOUR_DEPLOYMENT_ID" +#### if zhipuai from `https://open.bigmodel.cn`. You can set here or export API_KEY="YOUR_API_KEY" +# ZHIPUAI_API_KEY: "YOUR_API_KEY" + #### for Search ## Supported values: serpapi/google/serper/ddg diff --git a/docs/README_CN.md b/docs/README_CN.md index 9d6f34c11..038925184 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -12,14 +12,13 @@ CN doc EN doc JA doc -Discord Follow +Discord Follow License: MIT roadmap Twitter Follow

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face @@ -33,57 +32,35 @@

软件公司多角色示意图（正在逐步实现）

-## MetaGPT 的能力 - -https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 - - -## 示例（均由 GPT-4 生成） - -例如，键入`python startup.py "写个类似今日头条的推荐系统"`并回车，你会获得一系列输出，其一是数据结构与API设计 - -![今日头条 Recsys 数据 & API 设计](resources/workspace/content_rec_sys/resources/data_api_design.png) +## 安装 +### Pip安装 -这需要大约**0.2美元**（GPT-4 API的费用）来生成一个带有分析和设计的示例，大约2.0美元用于一个完整的项目 +```bash +# 第 1 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： +# 可以使用conda来初始化新的python环境 +# conda create -n metagpt python=3.9 +# conda activate metagpt +python3 --version -## 安装 +# 第 2 步：克隆最新仓库到您的本地机器，并进行安装。 +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT +pip3 install -e. # 或者 pip3 install metagpt # 安装稳定版本 -### 传统安装 +# 第 3 步：执行startup.py +# 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY +python3 startup.py "Write a cli snake game" -```bash -# 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js +# 第 4 步【可选的】：如果你想在执行过程中保存像象限图、系统设计、序列流程等图表这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 +# 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js npm --version sudo npm install -g @mermaid-js/mermaid-cli - -# 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： -python --version - -# 第 3 步：克隆仓库到您的本地机器，并进行安装。 -git clone https://github.com/geekan/metagpt -cd metagpt -pip install -e. ``` -**注意：** - -- 如果已经安装了Chrome、Chromium或MS Edge，可以通过将环境变量`PUPPETEER_SKIP_CHROMIUM_DOWNLOAD`设置为`true`来跳过下载Chromium。 - -- 一些人在全局安装此工具时遇到问题。在本地安装是替代解决方案， - - ```bash - npm install @mermaid-js/mermaid-cli - ``` - -- 不要忘记在config.yml中为mmdc配置配置， - - ```yml - PUPPETEER_CONFIG: "./config/puppeteer-config.json" - MMDC: "./node_modules/.bin/mmdc" - ``` - -- 如果`pip install -e.`失败并显示错误`[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`，请尝试使用`pip install -e. --user`运行。 +详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) ### Docker安装 +> 注意：在Windows中，你需要将 "/opt/metagpt" 替换为Docker具有创建权限的目录，比如"D:\Users\x\metagpt" ```bash # 步骤1: 下载metagpt官方镜像并准备好config.yaml @@ -99,121 +76,41 @@ docker run --rm \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ python startup.py "Write a cli snake game" - -# 您也可以启动一个容器并在其中执行命令 -docker run --name metagpt -d \ - --privileged \ - -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ - -v /opt/metagpt/workspace:/app/metagpt/workspace \ - metagpt/metagpt:latest - -docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" ``` -`docker run ...`做了以下事情: +详细的安装请安装 [docker_install](https://docs.deepwisdom.ai/zhcn/guide/get_started/installation.html#%E4%BD%BF%E7%94%A8docker%E5%AE%89%E8%A3%85) -- 以特权模式运行，有权限运行浏览器 -- 将主机文件 `/opt/metagpt/config/key.yaml` 映射到容器文件 `/app/metagpt/config/key.yaml` -- 将主机目录 `/opt/metagpt/workspace` 映射到容器目录 `/app/metagpt/workspace` -- 执行示例命令 `python startup.py "Write a cli snake game"` +### 快速开始的演示视频 +- 在 [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) 上进行体验 +- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) +- [官方演示视频](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) -### 自己构建镜像 - -```bash -# 您也可以自己构建metagpt镜像 -git clone https://github.com/geekan/MetaGPT.git -cd MetaGPT && docker build -t metagpt:custom . -``` - -## 配置 - -- 在 `config/key.yaml / config/config.yaml / env` 中配置您的 `OPENAI_API_KEY` -- 优先级顺序：`config/key.yaml > config/config.yaml > env` - -```bash -# 复制配置文件并进行必要的修改 -cp config/config.yaml config/key.yaml -``` - -| 变量名 | config/key.yaml | env | -| ----------------------------------- | ----------------------------------------- | ----------------------------------------------- | -| OPENAI_API_KEY # 用您自己的密钥替换 | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # 可选 | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | - -## 示例：启动一个创业公司 - -```shell -python startup.py "写一个命令行贪吃蛇" -# 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 -python startup.py "写一个命令行贪吃蛇" --code_review True -``` - -运行脚本后，您可以在 `workspace/` 目录中找到您的新项目。 -### 平台或工具的倾向性 -可以在阐述需求时说明想要使用的平台或工具。 -例如： -```shell -python startup.py "写一个基于pygame的命令行贪吃蛇" -``` - -### 使用 - -``` -名称 - startup.py - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 - -概要 - startup.py IDEA - -描述 - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 - -位置参数 - IDEA - 类型: str - 您的创新想法，例如"写一个命令行贪吃蛇。" - -标志 - --investment=INVESTMENT - 类型: float - 默认值: 3.0 - 作为投资者，您有机会向这家AI公司投入一定的美元金额。 - --n_round=N_ROUND - 类型: int - 默认值: 5 - -备注 - 您也可以用`标志`的语法，来处理`位置参数` -``` - -### 代码实现 - -```python -from metagpt.software_company import SoftwareCompany -from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer - -async def startup(idea: str, investment: float = 3.0, n_round: int = 5): - """运行一个创业公司。做一个老板""" - company = SoftwareCompany() - company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) -``` +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 -你可以查看`examples`，其中有单角色（带知识库）的使用例子与仅LLM的使用例子。 +## 教程 +- 🗒 [在线文档](https://docs.deepwisdom.ai/zhcn/) +- 💻 [如何使用](https://docs.deepwisdom.ai/zhcn/guide/get_started/quickstart.html) +- 🔎 [MetaGPT的能力及应用场景](https://docs.deepwisdom.ai/zhcn/guide/get_started/introduction.html) +- 🛠 如何构建你自己的智能体？ + - [MetaGPT的使用和开发教程 | 智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/agent_101.html) + - [MetaGPT的使用和开发教程 | 多智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/multi_agent_101.html) +- 🧑‍💻 贡献 + - [开发路线图](ROADMAP.md) +- 🔖 示例 + - [辩论](https://docs.deepwisdom.ai/zhcn/guide/use_cases/multi_agent/debate.html) + - [调研员](https://docs.deepwisdom.ai/zhcn/guide/use_cases/agent/researcher.html) + - [票据助手](https://docs.deepwisdom.ai/zhcn/guide/use_cases/agent/receipt_assistant.html) +- ❓ [常见问题解答](https://docs.deepwisdom.ai/zhcn/guide/faq.html) -## 快速体验 -对一些用户来说，安装配置本地环境是有困难的，下面这些教程能够让你快速体验到MetaGPT的魅力。 +## 支持 -- [MetaGPT快速体验](https://deepwisdom.feishu.cn/wiki/Q8ycw6J9tiNXdHk66MRcIN8Pnlg) +### 加入我们 -可直接在Huggingface Space体验 +📢 加入我们的[Discord频道](https://discord.gg/ZRHeExS6xv)！ -- https://huggingface.co/spaces/deepwisdom/MetaGPT +期待在那里与您相见！🎉 -## 联系信息 +### 联系信息 如果您对这个项目有任何问题或反馈，欢迎联系我们。我们非常欢迎您的建议！ @@ -222,13 +119,17 @@ async def startup(idea: str, investment: float = 3.0, n_round: int = 5): 我们会在2-3个工作日内回复所有问题。 -## 演示 - -https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d +## 引用 -## 加入我们 +引用 [arXiv paper](https://arxiv.org/abs/2308.00352): -📢 加入我们的Discord频道！ -https://discord.gg/ZRHeExS6xv - -期待在那里与您相见！🎉 +```bibtex +@misc{hong2023metagpt, + title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, + author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + year={2023}, + eprint={2308.00352}, + archivePrefix={arXiv}, + primaryClass={cs.AI} +} +``` diff --git a/docs/README_JA.md b/docs/README_JA.md index 2b2c35a62..411d190b4 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -19,7 +19,6 @@

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face @@ -60,17 +59,22 @@ https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace ### 伝統的なインストール ```bash -# ステップ 1: NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） -npm --version -sudo npm install -g @mermaid-js/mermaid-cli - -# ステップ 2: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: +# ステップ 1: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: python --version -# ステップ 3: リポジトリをローカルマシンにクローンし、インストールする。 -git clone https://github.com/geekan/metagpt -cd metagpt +# ステップ 2: リポジトリをローカルマシンにクローンし、インストールする。 +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT pip install -e. + +# ステップ 3: startup.py を実行する +# config.yaml を key.yaml にコピーし、独自の OPENAI_API_KEY を設定します +python3 startup.py "Write a cli snake game" + +# ステップ 4 [オプション]: 実行中に PRD ファイルなどのアーティファクトを保存する場合は、ステップ 3 の前にこのステップを実行できます。デフォルトでは、フレームワークには互換性があり、この手順を実行しなくてもプロセス全体を完了できます。 +# NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） +npm --version +sudo npm install -g @mermaid-js/mermaid-cli ``` **注:** @@ -159,6 +163,7 @@ Chromium のダウンロードをスキップすることができます。 注: この方法は pdf エクスポートに対応していません。 ### Docker によるインストール +> Windowsでは、"/opt/metagpt"をDockerが作成する権限を持つディレクトリに置き換える必要があります。例えば、"D:\Users\x\metagpt"などです。 ```bash # ステップ 1: metagpt 公式イメージをダウンロードし、config.yaml を準備する @@ -270,12 +275,12 @@ python startup.py "pygame をベースとした cli ヘビゲームを書く" ### コードウォークスルー ```python -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer async def startup(idea: str, investment: float = 3.0, n_round: int = 5): """スタートアップを実行する。ボスになる。""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) company.invest(investment) company.start_project(idea) @@ -295,12 +300,12 @@ Hugging Face Space で試す ## 引用 -現時点では、[Arxiv 論文](https://arxiv.org/abs/2308.00352)を引用してください: +現時点では、[arXiv 論文](https://arxiv.org/abs/2308.00352)を引用してください: ```bibtex @misc{hong2023metagpt, - title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, - author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + title={MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework}, + author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Ceyao Zhang and Jinlin Wang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and Jürgen Schmidhuber}, year={2023}, eprint={2308.00352}, archivePrefix={arXiv}, diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index 87d7a9c76..be34e5e5e 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -9,6 +9,7 @@ import fire +from metagpt.llm import LLM from metagpt.actions import Action from metagpt.roles import Role from metagpt.schema import Message @@ -19,19 +20,10 @@ class SimpleWriteCode(Action): PROMPT_TEMPLATE = """ Write a python function that can {instruction} and provide two runnnable test cases. Return ```python your_code_here ``` with NO other texts, - example: - ```python - # function - def add(a, b): - return a + b - # test cases - print(add(1, 2)) - print(add(3, 4)) - ``` your code: """ - def __init__(self, name="SimpleWriteCode", context=None, llm=None): + def __init__(self, name: str = "SimpleWriteCode", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, instruction: str): @@ -51,8 +43,9 @@ def parse_code(rsp): code_text = match.group(1) if match else rsp return code_text + class SimpleRunCode(Action): - def __init__(self, name="SimpleRunCode", context=None, llm=None): + def __init__(self, name: str = "SimpleRunCode", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, code_text: str): @@ -61,6 +54,7 @@ async def run(self, code_text: str): logger.info(f"{code_result=}") return code_result + class SimpleCoder(Role): def __init__( self, @@ -73,16 +67,16 @@ def __init__( async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - todo = self._rc.todo + todo = self._rc.todo # todo will be SimpleWriteCode() - msg = self._rc.memory.get()[-1] # retrieve the latest memory - instruction = msg.content + msg = self.get_memories(k=1)[0] # find the most recent messages - code_text = await SimpleWriteCode().run(instruction) - msg = Message(content=code_text, role=self.profile, cause_by=todo) + code_text = await todo.run(msg.content) + msg = Message(content=code_text, role=self.profile, cause_by=type(todo)) return msg + class RunnableCoder(Role): def __init__( self, @@ -92,43 +86,23 @@ def __init__( ): super().__init__(name, profile, **kwargs) self._init_actions([SimpleWriteCode, SimpleRunCode]) - - async def _think(self) -> None: - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None + self._set_react_mode(react_mode="by_order") async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") + # By choosing the Action by order under the hood + # todo will be first SimpleWriteCode() then SimpleRunCode() todo = self._rc.todo - msg = self._rc.memory.get()[-1] - - if isinstance(todo, SimpleWriteCode): - instruction = msg.content - result = await SimpleWriteCode().run(instruction) - elif isinstance(todo, SimpleRunCode): - code_text = msg.content - result = await SimpleRunCode().run(code_text) + msg = self.get_memories(k=1)[0] # find the most k recent messages + result = await todo.run(msg.content) - msg = Message(content=result, role=self.profile, cause_by=todo) + msg = Message(content=result, role=self.profile, cause_by=type(todo)) self._rc.memory.add(msg) return msg - async def _react(self) -> Message: - while True: - await self._think() - if self._rc.todo is None: - break - await self._act() - return Message(content="All job done", role=self.profile) -def main(msg="write a function that calculates the sum of a list"): +def main(msg="write a function that calculates the product of a list and run it"): # role = SimpleCoder() role = RunnableCoder() logger.info(msg) diff --git a/examples/debate.py b/examples/debate.py index 05db28070..a37e60848 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -7,14 +7,14 @@ import platform import fire -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.actions import Action, BossRequirement from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger -class ShoutOut(Action): - """Action: Shout out loudly in a debate (quarrel)""" +class SpeakAloud(Action): + """Action: Speak out aloud in a debate (quarrel)""" PROMPT_TEMPLATE = """ ## BACKGROUND @@ -27,7 +27,7 @@ class ShoutOut(Action): craft a strong and emotional response in 80 words, in {name}'s rhetoric and viewpoints, your will argue: """ - def __init__(self, name="ShoutOut", context=None, llm=None): + def __init__(self, name="SpeakAloud", context=None, llm=None): super().__init__(name, context, llm) async def run(self, context: str, name: str, opponent_name: str): @@ -39,96 +39,57 @@ async def run(self, context: str, name: str, opponent_name: str): return rsp -class Trump(Role): +class Debator(Role): def __init__( self, - name: str = "Trump", - profile: str = "Republican", + name: str, + profile: str, + opponent_name: str, **kwargs, ): super().__init__(name, profile, **kwargs) - self._init_actions([ShoutOut]) - self._watch([ShoutOut]) - self.name = "Trump" - self.opponent_name = "Biden" + self._init_actions([SpeakAloud]) + self._watch([BossRequirement, SpeakAloud]) + self.name = name + self.opponent_name = opponent_name async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] + self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] return len(self._rc.news) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") + todo = self._rc.todo # An instance of SpeakAloud - msg_history = self._rc.memory.get_by_actions([ShoutOut]) - context = [] - for m in msg_history: - context.append(str(m)) - context = "\n".join(context) + memories = self.get_memories() + context = "\n".join(f"{msg.sent_from}: {msg.content}" for msg in memories) + # print(context) - rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) + rsp = await todo.run(context=context, name=self.name, opponent_name=self.opponent_name) msg = Message( content=rsp, role=self.profile, - cause_by=ShoutOut, + cause_by=type(todo), sent_from=self.name, send_to=self.opponent_name, ) - return msg - -class Biden(Role): - def __init__( - self, - name: str = "Biden", - profile: str = "Democrat", - **kwargs, - ): - super().__init__(name, profile, **kwargs) - self._init_actions([ShoutOut]) - self._watch([BossRequirement, ShoutOut]) - self.name = "Biden" - self.opponent_name = "Trump" - - async def _observe(self) -> int: - await super()._observe() - # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, - # disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.cause_by == BossRequirement or msg.send_to == self.name] - return len(self._rc.news) - - async def _act(self) -> Message: - logger.info(f"{self._setting}: ready to {self._rc.todo}") - - msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) - context = [] - for m in msg_history: - context.append(str(m)) - context = "\n".join(context) - - rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) - - msg = Message( - content=rsp, - role=self.profile, - cause_by=ShoutOut, - sent_from=self.name, - send_to=self.opponent_name, - ) + self._rc.memory.add(msg) return msg -async def startup(idea: str, investment: float = 3.0, n_round: int = 5, - code_review: bool = False, run_tests: bool = False): - """We reuse the startup paradigm for roles to interact with each other. - Now we run a startup of presidents and watch they quarrel. :) """ - company = SoftwareCompany() - company.hire([Biden(), Trump()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) +async def debate(idea: str, investment: float = 3.0, n_round: int = 5): + """Run a team of presidents and watch they quarrel. :) """ + Biden = Debator(name="Biden", profile="Democrat", opponent_name="Trump") + Trump = Debator(name="Trump", profile="Republican", opponent_name="Biden") + team = Team() + team.hire([Biden, Trump]) + team.invest(investment) + team.start_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first + await team.run(n_round=n_round) def main(idea: str, investment: float = 3.0, n_round: int = 10): @@ -141,7 +102,7 @@ def main(idea: str, investment: float = 3.0, n_round: int = 10): """ if platform.system() == "Windows": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - asyncio.run(startup(idea, investment, n_round)) + asyncio.run(debate(idea, investment, n_round)) if __name__ == '__main__': diff --git a/metagpt/config.py b/metagpt/config.py index 27455d38d..3f9e742bd 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -45,10 +45,11 @@ def __init__(self, yaml_file=default_yaml_file): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") - if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key - ): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") + self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY") + if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and \ + (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and \ + (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key): + raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy if openai_proxy: diff --git a/metagpt/const.py b/metagpt/const.py index 7f3f87dfa..407ce803a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -6,7 +6,7 @@ @File : const.py """ from pathlib import Path - +from loguru import logger def get_project_root(): """Search upwards to find the project root directory.""" @@ -17,10 +17,15 @@ def get_project_root(): or (current_path / ".project_root").exists() or (current_path / ".gitignore").exists() ): + # use metagpt with git clone will land here + logger.info(f"PROJECT_ROOT set to {str(current_path)}") return current_path parent_path = current_path.parent if parent_path == current_path: - raise Exception("Project root not found.") + # use metagpt with pip install will land here + cwd = Path.cwd() + logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") + return cwd current_path = parent_path diff --git a/metagpt/llm.py b/metagpt/llm.py index e6f815950..4edcd7a83 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -6,14 +6,27 @@ @File : llm.py """ +from metagpt.logs import logger +from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM +from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +from metagpt.provider.spark_api import SparkAPI +from metagpt.provider.human_provider import HumanProvider -DEFAULT_LLM = LLM() -CLAUDE_LLM = Claude() -async def ai_func(prompt): - """使用LLM进行QA - QA with LLMs - """ - return await DEFAULT_LLM.aask(prompt) +def LLM() -> "BaseGPTAPI": + """ initialize different LLM instance according to the key field existence""" + # TODO a little trick, can use registry to initialize LLM instance further + if CONFIG.openai_api_key: + llm = OpenAIGPTAPI() + elif CONFIG.claude_api_key: + llm = Claude() + elif CONFIG.spark_api_key: + llm = SparkAPI() + elif CONFIG.zhipuai_api_key: + llm = ZhiPuAIGPTAPI() + else: + raise RuntimeError("You should config a LLM configuration first") + + return llm diff --git a/metagpt/provider/base_chatbot.py b/metagpt/provider/base_chatbot.py index abdf423f4..72e6c94f9 100644 --- a/metagpt/provider/base_chatbot.py +++ b/metagpt/provider/base_chatbot.py @@ -13,6 +13,7 @@ class BaseChatbot(ABC): """Abstract GPT class""" mode: str = "API" + use_system_prompt: bool = True @abstractmethod def ask(self, msg: str) -> str: diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index de61167b9..b6b034329 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : base_gpt_api.py """ +import json from abc import abstractmethod from typing import Optional @@ -14,7 +15,8 @@ class BaseGPTAPI(BaseChatbot): """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities""" - system_prompt = 'You are a helpful assistant.' + + system_prompt = "You are a helpful assistant." def _user_msg(self, msg: str) -> dict[str, str]: return {"role": "user", "content": msg} @@ -32,15 +34,17 @@ def _default_system_msg(self): return self._system_msg(self.system_prompt) def ask(self, msg: str) -> str: - message = [self._default_system_msg(), self._user_msg(msg)] + message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] rsp = self.completion(message) return self.get_choice_text(rsp) async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: if system_msgs: - message = self._system_msgs(system_msgs) + [self._user_msg(msg)] + message = self._system_msgs(system_msgs) + [self._user_msg(msg)] if self.use_system_prompt \ + else [self._user_msg(msg)] else: - message = [self._default_system_msg(), self._user_msg(msg)] + message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt \ + else [self._user_msg(msg)] rsp = await self.acompletion_text(message, stream=True) logger.debug(message) # logger.debug(rsp) @@ -108,11 +112,50 @@ def get_choice_text(self, rsp: dict) -> str: """Required to provide the first text of choice""" return rsp.get("choices")[0]["message"]["content"] + def get_choice_function(self, rsp: dict) -> dict: + """Required to provide the first function of choice + :param dict rsp: OpenAI chat.comletion respond JSON, Note "message" must include "tool_calls", + and "tool_calls" must include "function", for example: + {... + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_Y5r6Ddr2Qc2ZrqgfwzPX5l72", + "type": "function", + "function": { + "name": "execute", + "arguments": "{\n \"language\": \"python\",\n \"code\": \"print('Hello, World!')\"\n}" + } + } + ] + }, + "finish_reason": "stop" + } + ], + ...} + :return dict: return first function of choice, for exmaple, + {'name': 'execute', 'arguments': '{\n "language": "python",\n "code": "print(\'Hello, World!\')"\n}'} + """ + return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"].to_dict() + + def get_choice_function_arguments(self, rsp: dict) -> dict: + """Required to provide the first function arguments of choice. + + :param dict rsp: same as in self.get_choice_function(rsp) + :return dict: return the first function arguments of choice, for example, + {'language': 'python', 'code': "print('Hello, World!')"} + """ + return json.loads(self.get_choice_function(rsp)["arguments"]) + def messages_to_prompt(self, messages: list[dict]): """[{"role": "user", "content": msg}] to user: etc.""" - return '\n'.join([f"{i['role']}: {i['content']}" for i in messages]) + return "\n".join([f"{i['role']}: {i['content']}" for i in messages]) def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" return [i.to_dict() for i in messages] - \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6ebed2c16..34e5693f8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -21,6 +21,8 @@ from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE +from metagpt.schema import Message from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( TOKEN_COSTS, @@ -110,7 +112,6 @@ def get_total_completion_tokens(self): """ return self.total_completion_tokens - def get_total_cost(self): """ Get the total cost of API calls. @@ -120,7 +121,6 @@ def get_total_cost(self): """ return self.total_cost - def get_costs(self) -> Costs: """Get all costs""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) @@ -181,7 +181,7 @@ async def _achat_completion_stream(self, messages: list[dict]) -> str: self._update_costs(usage) return full_reply_content - def _cons_kwargs(self, messages: list[dict]) -> dict: + def _cons_kwargs(self, messages: list[dict], **configs) -> dict: kwargs = { "messages": messages, "max_tokens": self.get_max_tokens(messages), @@ -190,6 +190,9 @@ def _cons_kwargs(self, messages: list[dict]) -> dict: "temperature": 0.3, "timeout": 3, } + if configs: + kwargs.update(configs) + if CONFIG.openai_api_type == "azure": if CONFIG.deployment_name and CONFIG.deployment_id: raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model") @@ -239,6 +242,81 @@ async def acompletion_text(self, messages: list[dict], stream=False) -> str: rsp = await self._achat_completion(messages) return self.get_choice_text(rsp) + def _func_configs(self, messages: list[dict], **kwargs) -> dict: + """ + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create + """ + if "tools" not in kwargs: + configs = { + "tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}], + "tool_choice": GENERAL_TOOL_CHOICE, + } + kwargs.update(configs) + + return self._cons_kwargs(messages, **kwargs) + + def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict: + rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs)) + self._update_costs(rsp.get("usage")) + return rsp + + async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict: + rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs)) + self._update_costs(rsp.get("usage")) + return rsp + + def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: + """convert messages to list[dict].""" + if isinstance(messages, list): + messages = [Message(msg) if isinstance(msg, str) else msg for msg in messages] + return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages] + + if isinstance(messages, Message): + messages = [messages.to_dict()] + elif isinstance(messages, str): + messages = [{"role": "user", "content": messages}] + else: + raise ValueError( + f"Only support messages type are: str, Message, list[dict], but got {type(messages).__name__}!" + ) + return messages + + def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: + """Use function of tools to ask a code. + + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create + + Examples: + + >>> llm = OpenAIGPTAPI() + >>> llm.ask_code("Write a python hello world code.") + {'language': 'python', 'code': "print('Hello, World!')"} + >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] + >>> llm.ask_code(msg) + {'language': 'python', 'code': "print('Hello, World!')"} + """ + messages = self._process_message(messages) + rsp = self._chat_completion_function(messages, **kwargs) + return self.get_choice_function_arguments(rsp) + + async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: + """Use function of tools to ask a code. + + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create + + Examples: + + >>> llm = OpenAIGPTAPI() + >>> rsp = await llm.ask_code("Write a python hello world code.") + >>> rsp + {'language': 'python', 'code': "print('Hello, World!')"} + >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] + >>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + """ + messages = self._process_message(messages) + rsp = await self._achat_completion_function(messages, **kwargs) + return self.get_choice_function_arguments(rsp) + def _calc_usage(self, messages: list[dict], rsp: str) -> dict: usage = {} if CONFIG.calc_usage: diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 6d65575a8..1f6685b38 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -207,6 +207,7 @@ async def _act_sp_precision(self) -> Message: async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" + logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: return await self._act_sp_precision() return await self._act_sp() diff --git a/metagpt/roles/invoice_ocr_assistant.py b/metagpt/roles/invoice_ocr_assistant.py index c307b20c0..15f831c97 100644 --- a/metagpt/roles/invoice_ocr_assistant.py +++ b/metagpt/roles/invoice_ocr_assistant.py @@ -42,17 +42,7 @@ def __init__( self.filename = "" self.origin_query = "" self.orc_data = None - - async def _think(self) -> None: - """Determine the next action to be taken by the role.""" - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None + self._set_react_mode(react_mode="by_order") async def _act(self) -> Message: """Perform an action as determined by the role. @@ -94,17 +84,3 @@ async def _act(self) -> Message: msg = Message(content=content, instruct_content=resp) self._rc.memory.add(msg) return msg - - async def _react(self) -> Message: - """Execute the invoice ocr assistant's think and actions. - - Returns: - A message containing the final result of the assistant's actions. - """ - while True: - await self._think() - if self._rc.todo is None: - break - msg = await self._act() - return msg - diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index acb46c718..c5512121a 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -31,20 +31,11 @@ def __init__( ): super().__init__(name, profile, goal, constraints, **kwargs) self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) + self._set_react_mode(react_mode="by_order") self.language = language if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") - async def _think(self) -> None: - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None - async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo @@ -73,12 +64,8 @@ async def _act(self) -> Message: self._rc.memory.add(ret) return ret - async def _react(self) -> Message: - while True: - await self._think() - if self._rc.todo is None: - break - msg = await self._act() + async def react(self) -> Message: + msg = await super().react() report = msg.instruct_content self.write_report(report.topic, report.content) return msg diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 44bb3e976..b96c361c0 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -7,14 +7,15 @@ """ from __future__ import annotations -from typing import Iterable, Type +from typing import Iterable, Type, Union +from enum import Enum from pydantic import BaseModel, Field # from metagpt.environment import Environment from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput -from metagpt.llm import LLM +from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message @@ -27,12 +28,14 @@ {history} === -You can now choose one of the following stages to decide the stage you need to go in the next step: +Your previous stage: {previous_state} + +Now choose one of the following stages you need to go to in the next step: {states} Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation. Please note that the answer only needs a number, no need to add any other text. -If there is no conversation record, choose 0. +If you think you have completed your goal and don't need to go to any of the stages, return -1. Do not answer anything else, and do not add any other information in your answer. """ @@ -46,6 +49,14 @@ {name}: {result} """ +class RoleReactMode(str, Enum): + REACT = "react" + BY_ORDER = "by_order" + PLAN_AND_ACT = "plan_and_act" + + @classmethod + def values(cls): + return [item.value for item in cls] class RoleSetting(BaseModel): """Role Settings""" @@ -54,6 +65,7 @@ class RoleSetting(BaseModel): goal: str constraints: str desc: str + is_human: bool def __str__(self): return f"{self.name}({self.profile})" @@ -67,10 +79,12 @@ class RoleContext(BaseModel): env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) - state: int = Field(default=0) + state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) + react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + max_react_loop: int = 1 class Config: arbitrary_types_allowed = True @@ -93,9 +107,10 @@ def history(self) -> list[Message]: class Role: """Role/Agent""" - def __init__(self, name="", profile="", goal="", constraints="", desc=""): - self._llm = LLM() - self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) + def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): + self._llm = LLM() if not is_human else HumanProvider() + self._setting = RoleSetting(name=name, profile=profile, goal=goal, + constraints=constraints, desc=desc, is_human=is_human) self._states = [] self._actions = [] self._role_id = str(self._setting) @@ -109,24 +124,48 @@ def _init_actions(self, actions): self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): - i = action("") + i = action("", llm=self._llm) else: + if self._setting.is_human and not isinstance(action.llm, HumanProvider): + logger.warning(f"is_human attribute does not take effect," + f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) self._states.append(f"{idx}. {action}") + def _set_react_mode(self, react_mode: str, max_react_loop: int = 1): + """Set strategy of the Role reacting to observed Message. Variation lies in how + this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions. + + Args: + react_mode (str): Mode for choosing action during the _think stage, can be one of: + "react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically; + "by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...; + "plan_and_act": first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... + Use llm to come up with the plan dynamically. + Defaults to "react". + max_react_loop (int): Maximum react cycles to execute, used to prevent the agent from reacting forever. + Take effect only when react_mode is react, in which we use llm to choose actions, including termination. + Defaults to 1, i.e. _think -> _act (-> return result and end) + """ + assert react_mode in RoleReactMode.values(), f"react_mode must be one of {RoleReactMode.values()}" + self._rc.react_mode = react_mode + if react_mode == RoleReactMode.REACT: + self._rc.max_react_loop = max_react_loop + def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" self._rc.watch.update(actions) # check RoleContext after adding watch actions self._rc.check(self._role_id) - def _set_state(self, state): + def _set_state(self, state: int): """Update the current state.""" self._rc.state = state logger.debug(self._actions) - self._rc.todo = self._actions[self._rc.state] + self._rc.todo = self._actions[self._rc.state] if state >= 0 else None def set_env(self, env: 'Environment'): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" @@ -151,13 +190,19 @@ async def _think(self) -> None: return prompt = self._get_prefix() prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + n_states=len(self._states) - 1, previous_state=self._rc.state) + # print(prompt) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") - if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') - next_state = "0" - self._set_state(int(next_state)) + if (not next_state.isdigit() and next_state != "-1") \ + or int(next_state) not in range(-1, len(self._states)): + logger.warning(f'Invalid answer of state, {next_state=}, will be set to -1') + next_state = -1 + else: + next_state = int(next_state) + if next_state == -1: + logger.info(f"End actions with {next_state=}") + self._set_state(next_state) async def _act(self) -> Message: # prompt = self.get_prefix() @@ -203,10 +248,45 @@ def _publish_message(self, msg): self._rc.env.publish_message(msg) async def _react(self) -> Message: - """Think first, then act""" - await self._think() - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - return await self._act() + """Think first, then act, until the Role _think it is time to stop and requires no more todo. + This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically + """ + actions_taken = 0 + rsp = Message("No actions taken yet") # will be overwritten after Role _act + while actions_taken < self._rc.max_react_loop: + # think + await self._think() + if self._rc.todo is None: + break + # act + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + rsp = await self._act() + actions_taken += 1 + return rsp # return output from the last action + + async def _act_by_order(self) -> Message: + """switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...""" + for i in range(len(self._states)): + self._set_state(i) + rsp = await self._act() + return rsp # return output from the last action + + async def _plan_and_act(self) -> Message: + """first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically.""" + # TODO: to be implemented + return Message("") + + async def react(self) -> Message: + """Entry to one of three strategies by which Role reacts to the observed Message""" + if self._rc.react_mode == RoleReactMode.REACT: + rsp = await self._react() + elif self._rc.react_mode == RoleReactMode.BY_ORDER: + rsp = await self._act_by_order() + elif self._rc.react_mode == RoleReactMode.PLAN_AND_ACT: + rsp = await self._plan_and_act() + self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None + return rsp def recv(self, message: Message) -> None: """add message to history.""" @@ -223,6 +303,10 @@ async def handle(self, message: Message) -> Message: return await self._react() + def get_memories(self, k=0) -> list[Message]: + """A wrapper to return the most recent k memories of this role, return all when k=0""" + return self._rc.memory.get(k=k) + async def run(self, message=None): """Observe, and think and act based on the results of the observation""" if message: @@ -237,7 +321,7 @@ async def run(self, message=None): logger.debug(f"{self._setting}: no news. waiting.") return - rsp = await self._react() + rsp = await self.react() # Publish the reply to the environment, waiting for the next subscriber to process self._publish_message(rsp) return rsp diff --git a/metagpt/software_company.py b/metagpt/software_company.py index b2bd18c58..d44a0068a 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -5,58 +5,9 @@ @Author : alexanderwu @File : software_company.py """ -from pydantic import BaseModel, Field +from metagpt.team import Team as SoftwareCompany -from metagpt.actions import BossRequirement -from metagpt.config import CONFIG -from metagpt.environment import Environment -from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import NoMoneyException - - -class SoftwareCompany(BaseModel): - """ - Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging, - dedicated to writing executable code. - """ - environment: Environment = Field(default_factory=Environment) - investment: float = Field(default=10.0) - idea: str = Field(default="") - - class Config: - arbitrary_types_allowed = True - - def hire(self, roles: list[Role]): - """Hire roles to cooperate""" - self.environment.add_roles(roles) - - def invest(self, investment: float): - """Invest company. raise NoMoneyException when exceed max_budget.""" - self.investment = investment - CONFIG.max_budget = investment - logger.info(f'Investment: ${investment}.') - - def _check_balance(self): - if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - - def start_project(self, idea): - """Start a project from publishing boss requirement.""" - self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) - - def _save(self): - logger.info(self.json()) - - async def run(self, n_round=3): - """Run company until target round or no money""" - while n_round > 0: - # self._save() - n_round -= 1 - logger.debug(f"{n_round=}") - self._check_balance() - await self.environment.run() - return self.environment.history - \ No newline at end of file +import warnings +warnings.warn("metagpt.software_company is deprecated and will be removed in the future" + "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", + DeprecationWarning, 2) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 5e5b275b0..204c22c67 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -34,7 +34,10 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, engine = CONFIG.mermaid_engine.lower() if engine == "nodejs": if check_cmd_exists(CONFIG.mmdc) != 0: - logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + logger.warning( + "RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc," + "or consider changing MERMAID_ENGINE to `playwright`, `pyppeteer`, or `ink`." + ) return -1 for suffix in ["pdf", "svg", "png"]: diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index a5a65803a..1af96f272 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -22,6 +22,7 @@ "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12}, "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, + "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -37,6 +38,7 @@ "gpt-4-32k-0314": 32768, "gpt-4-0613": 8192, "text-embedding-ada-002": 8192, + "chatglm_turbo": 32768 } @@ -68,7 +70,9 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): return count_message_tokens(messages, model="gpt-4-0613") else: raise NotImplementedError( - f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""" + f"num_tokens_from_messages() is not implemented for model {model}. " + f"See https://github.com/openai/openai-python/blob/main/chatml.md " + f"for information on how messages are converted to tokens." ) num_tokens = 0 for message in messages: diff --git a/requirements.txt b/requirements.txt index 24a2d94c3..f0169d7fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ langchain==0.0.231 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 -openai +openai>=0.28.0 openpyxl beautifulsoup4==4.12.2 pandas==2.0.3 @@ -44,4 +44,4 @@ ta==0.10.2 semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 - +zhipuai==1.0.7 diff --git a/setup.py b/setup.py index f9ae768e6..239156ae3 100644 --- a/setup.py +++ b/setup.py @@ -30,16 +30,16 @@ def run(self): setup( name="metagpt", - version="0.1", + version="0.3.0", description="The Multi-Role Meta Programming Framework", long_description=long_description, long_description_content_type="text/markdown", - url="https://gitlab.deepwisdomai.com/pub/metagpt", + url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@fuzhi.ai", license="Apache 2.0", keywords="metagpt multi-role multi-agent programming gpt llm", - packages=find_packages(exclude=["contrib", "docs", "examples"]), + packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, extras_require={ diff --git a/startup.py b/startup.py index e2a903c9b..e9fbf94d3 100644 --- a/startup.py +++ b/startup.py @@ -11,7 +11,7 @@ ProjectManager, QaEngineer, ) -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team async def startup( @@ -23,7 +23,7 @@ async def startup( implement: bool = True, ): """Run a startup. Be a boss.""" - company = SoftwareCompany() + company = Team() company.hire( [ ProductManager(), diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index 285bff323..d58d31bd9 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -2,7 +2,7 @@ # @Date : 2023/7/22 02:40 # @Author : stellahong (stellahong@fuzhi.ai) # -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProductManager from tests.metagpt.roles.ui_role import UI @@ -15,7 +15,7 @@ def test_add_ui(): async def test_ui_role(idea: str, investment: float = 3.0, n_round: int = 5): """Run a startup. Be a boss.""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), UI()]) company.invest(investment) company.start_project(idea) diff --git a/tests/metagpt/test_software_company.py b/tests/metagpt/test_software_company.py index 00538442c..4fc651f52 100644 --- a/tests/metagpt/test_software_company.py +++ b/tests/metagpt/test_software_company.py @@ -8,12 +8,12 @@ import pytest from metagpt.logs import logger -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team @pytest.mark.asyncio -async def test_software_company(): - company = SoftwareCompany() +async def test_team(): + company = Team() company.start_project("做一个基础搜索引擎，可以支持知识库") history = await company.run(n_round=5) logger.info(history) From 45e48c8093e4bae519f62adc3b51e3c74ada1976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 17:44:16 +0800 Subject: [PATCH 168/232] feat: merge geekan:main --- .pre-commit-config.yam_ => .pre-commit-config.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .pre-commit-config.yam_ => .pre-commit-config.yaml (100%) diff --git a/.pre-commit-config.yam_ b/.pre-commit-config.yaml similarity index 100% rename from .pre-commit-config.yam_ rename to .pre-commit-config.yaml From ef9a925281e0a06ab910d14dcd5bc48a9689cc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:11:12 +0800 Subject: [PATCH 169/232] feat: + gitpython 3.1.40 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 73a03d537..e72efc76c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,3 +45,4 @@ semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 aiofiles==23.2.1 +gitpython==3.1.40 \ No newline at end of file From 4c99107a333d6e9dc6bb52399de578364002ac4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:23:20 +0800 Subject: [PATCH 170/232] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/debug_error.py | 7 +++--- metagpt/utils/file_repository.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index e4a15d38d..dd1527154 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -16,6 +16,7 @@ from metagpt.logs import logger from metagpt.schema import RunCodeResult from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository PROMPT_TEMPLATE = """ NOTICE @@ -50,7 +51,7 @@ def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) async def run(self, *args, **kwargs) -> str: - output_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get(self.context.output_filename) + output_doc = await FileRepository.get_file(filename=self.context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO) if not output_doc: return "" output_detail = RunCodeResult.loads(output_doc.content) @@ -60,10 +61,10 @@ async def run(self, *args, **kwargs) -> str: return "" logger.info(f"Debug and rewrite {self.context.code_filename}") - code_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(self.context.code_filename) + code_doc = await FileRepository.get_file(filename=self.context.code_filename, relative_path=CONFIG.src_workspace) if not code_doc: return "" - test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(self.context.test_filename) + test_doc = await FileRepository.get_file(filename=self.context.test_filename, relative_path=TEST_CODES_FILE_REPO) if not test_doc: return "" prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 8de4bdf5b..3df53cca3 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -16,6 +16,7 @@ import aiofiles +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Document from metagpt.utils.json_to_markdown import json_to_markdown @@ -186,3 +187,44 @@ async def save_pdf(self, doc: Document): filename = Path(doc.filename).with_suffix(".md") await self.save(filename=str(filename), content=json_to_markdown(m)) logger.info(f"File Saved: {str(filename)}") + + @staticmethod + async def get_file(filename: Path | str, relative_path: Path | str = ".") -> Document | None: + """Retrieve a specific file from the file repository. + + :param filename: The name or path of the file to retrieve. + :type filename: Path or str + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: The document representing the file, or None if not found. + :rtype: Document or None + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.get(filename=filename) + + @staticmethod + async def get_all_files(relative_path: Path | str = ".") -> List[Document]: + """Retrieve all files from the file repository. + + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: A list of documents representing all files in the repository. + :rtype: List[Document] + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.get_all() + + @staticmethod + async def save_file(filename: Path | str, content, dependencies: List[str] = None, relative_path: Path | str = "."): + """Save a file to the file repository. + + :param filename: The name or path of the file to save. + :type filename: Path or str + :param content: The content of the file. + :param dependencies: A list of dependencies for the file. + :type dependencies: List[str], optional + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.save(filename=filename, content=content, dependencies=dependencies) From 81e719faa21bd08d3f97545c30d48658348afc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:29:29 +0800 Subject: [PATCH 171/232] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/prepare_documents.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index c9b60ff27..92d5730b2 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -24,8 +24,7 @@ def __init__(self, name="", context=None, llm=None): async def run(self, with_messages, **kwargs): if CONFIG.git_repo: - docs_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) - doc = await docs_repo.get(REQUIREMENT_FILENAME) + doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) # Create and initialize the workspace folder, initialize the Git environment. @@ -34,9 +33,8 @@ async def run(self, with_messages, **kwargs): CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. - docs_file_repository = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) - await docs_file_repository.save(REQUIREMENT_FILENAME, content=doc.content) + await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=doc.content, relative_path=DOCS_FILE_REPO) # Send a Message notification to the WritePRD action, instructing it to process requirements using # `docs/requirement.txt` and `docs/prds/`. From fa675ea3157a4a4a1b09209f52e4714d7a5e60d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:32:33 +0800 Subject: [PATCH 172/232] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/project_management.py | 9 ++++----- metagpt/const.py | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 641d21533..d679a730c 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -15,7 +15,7 @@ from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO, PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.get_template import get_template @@ -263,16 +263,15 @@ async def _update_requirements(doc): m = json.loads(doc.content) packages = set(m.get("Required Python third-party packages", set())) file_repo = CONFIG.git_repo.new_file_repository() - filename = "requirements.txt" - requirement_doc = await file_repo.get(filename) + requirement_doc = await file_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME) if not requirement_doc: - requirement_doc = Document(filename=filename, root_path=".", content="") + requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="") lines = requirement_doc.content.splitlines() for pkg in lines: if pkg == "": continue packages.add(pkg) - await file_repo.save(filename, content="\n".join(packages)) + await file_repo.save(PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages)) @staticmethod async def _save_pdf(task_doc): diff --git a/metagpt/const.py b/metagpt/const.py index a8c7356ca..ce06655f1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -53,6 +53,8 @@ def get_project_root(): MESSAGE_ROUTE_TO_NONE = "" REQUIREMENT_FILENAME = "requirement.txt" +PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt" + DOCS_FILE_REPO = "docs" PRDS_FILE_REPO = "docs/prds" SYSTEM_DESIGN_FILE_REPO = "docs/system_design" From 726eadf1cce7205fbbe960d30326c5eb118c09a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:54:41 +0800 Subject: [PATCH 173/232] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/design_api.py | 4 +-- metagpt/actions/prepare_documents.py | 2 +- metagpt/actions/project_management.py | 4 +-- metagpt/actions/write_code.py | 6 ++--- metagpt/actions/write_prd.py | 8 +++--- metagpt/utils/file_repository.py | 39 +++++++++++++++++++++++---- 6 files changed, 45 insertions(+), 18 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 021edfe72..f987c6042 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -24,6 +24,7 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -304,8 +305,7 @@ async def _save_seq_flow(design_doc): @staticmethod async def _save_pdf(design_doc): - file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) - await file_repo.save_pdf(doc=design_doc) + await FileRepository.save_as(doc=design_doc, with_suffix=".md", relative_path=SYSTEM_DESIGN_PDF_FILE_REPO) @staticmethod async def _save_mermaid_file(data: str, pathname: Path): diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 92d5730b2..30558c93f 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -29,7 +29,7 @@ async def run(self, with_messages, **kwargs): # Create and initialize the workspace folder, initialize the Git environment. CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_file_name() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index d679a730c..7205d11e7 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -18,6 +18,7 @@ from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO, PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template templates = { @@ -275,8 +276,7 @@ async def _update_requirements(doc): @staticmethod async def _save_pdf(task_doc): - file_repo = CONFIG.git_repo.new_file_repository(TASK_PDF_FILE_REPO) - await file_repo.save_pdf(doc=task_doc) + await FileRepository.save_as(doc=task_doc, with_suffix=".md", relative_path=TASK_PDF_FILE_REPO) class AssignTasks(Action): diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index e9d41bb20..3a4ca7768 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -23,6 +23,7 @@ from metagpt.logs import logger from metagpt.schema import CodingContext, RunCodeResult from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository PROMPT_TEMPLATE = """ NOTICE @@ -82,9 +83,8 @@ async def write_code(self, prompt) -> str: async def run(self, *args, **kwargs) -> CodingContext: coding_context = CodingContext.loads(self.context.content) - test_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get( - "test_" + coding_context.filename + ".json" - ) + test_doc = await FileRepository.get_file(filename="test_" + coding_context.filename + ".json", + relative_path=TEST_OUTPUTS_FILE_REPO) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index cc21058b4..c1653a850 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -290,8 +290,7 @@ def __init__(self, name="", context=None, llm=None): async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are # related to the PRD. If they are related, rewrite the PRD. - docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) - requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) + requirement_doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) prd_docs = await prds_file_repo.get_all() change_files = Documents() @@ -355,7 +354,7 @@ async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **k prd = await self._run_new_requirement(requirements=[requirement_doc.content], *args, **kwargs) new_prd_doc = Document( root_path=PRDS_FILE_REPO, - filename=FileRepository.new_file_name() + ".json", + filename=FileRepository.new_filename() + ".json", content=prd.instruct_content.json(), ) elif await self._is_relative_to(requirement_doc, prd_doc): @@ -382,5 +381,4 @@ async def _save_competitive_analysis(prd_doc): @staticmethod async def _save_pdf(prd_doc): - file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) - await file_repo.save_pdf(doc=prd_doc) + await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 3df53cca3..018cac168 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -168,7 +168,7 @@ def get_change_dir_files(self, dir: Path | str) -> List: return children @staticmethod - def new_file_name(): + def new_filename(): """Generate a new filename based on the current timestamp and a UUID suffix. :return: A new filename string. @@ -178,14 +178,22 @@ def new_file_name(): # guid_suffix = str(uuid.uuid4())[:8] # return f"{current_time}x{guid_suffix}" - async def save_pdf(self, doc: Document): - """Save a Document as a PDF file. + async def save_doc(self, doc: Document, with_suffix:str = None, dependencies: List[str] = None): + """Save a Document instance as a PDF file. + + This method converts the content of the Document instance to Markdown, + saves it to a file with an optional specified suffix, and logs the saved file. :param doc: The Document instance to be saved. + :type doc: Document + :param with_suffix: An optional suffix to append to the saved file's name. + :type with_suffix: str, optional + :param dependencies: A list of dependencies for the saved file. + :type dependencies: List[str], optional """ m = json.loads(doc.content) - filename = Path(doc.filename).with_suffix(".md") - await self.save(filename=str(filename), content=json_to_markdown(m)) + filename = Path(doc.filename).with_suffix(with_suffix) if with_suffix is not None else Path(doc.filename) + await self.save(filename=str(filename), content=json_to_markdown(m), dependencies=dependencies) logger.info(f"File Saved: {str(filename)}") @staticmethod @@ -228,3 +236,24 @@ async def save_file(filename: Path | str, content, dependencies: List[str] = Non """ file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) return await file_repo.save(filename=filename, content=content, dependencies=dependencies) + + @staticmethod + async def save_as(doc:Document, with_suffix:str = None, dependencies: List[str] = None, relative_path: Path | str = "."): + """Save a Document instance with optional modifications. + + This static method creates a new FileRepository, saves the Document instance + with optional modifications (such as a suffix), and logs the saved file. + + :param doc: The Document instance to be saved. + :type doc: Document + :param with_suffix: An optional suffix to append to the saved file's name. + :type with_suffix: str, optional + :param dependencies: A list of dependencies for the saved file. + :type dependencies: List[str], optional + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: A boolean indicating whether the save operation was successful. + :rtype: bool + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies) From 2cd7d266ddce6a4c0979e29363d38b6c58f9b15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 21:20:46 +0800 Subject: [PATCH 174/232] feat: merge Config class of send18:dev branch --- metagpt/actions/run_code.py | 6 +++-- metagpt/config.py | 52 ++++++++++++++++++++++++++++++------- metagpt/const.py | 3 +++ 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 1e7010e52..fa13a0980 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -12,13 +12,15 @@ RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. 4. According to section 2.2.3.5.7 of RFC 135, change the method of transferring file content (code files, unit test files, log files) from using the message to using the file name. + 5. Merged the `Config` class of send18:dev branch to take over the set/get operations of the Environment + class. """ -import os import subprocess import traceback from typing import Tuple from metagpt.actions.action import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import RunCodeResult @@ -92,7 +94,7 @@ async def run_script(cls, working_directory, additional_python_paths=[], command additional_python_paths = [str(path) for path in additional_python_paths] # Copy the current environment variables - env = os.environ.copy() + env = CONFIG.new_environ() # Modify the PYTHONPATH environment variable additional_python_paths = [working_directory] + additional_python_paths diff --git a/metagpt/config.py b/metagpt/config.py index a20f58ec1..1b70d5fa6 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -7,11 +7,13 @@ 2. Add the parameter `src_workspace` for the old version project path. """ import os +from copy import deepcopy +from typing import Any import openai import yaml -from metagpt.const import PROJECT_ROOT +from metagpt.const import OPTIONS, PROJECT_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -42,9 +44,11 @@ class Config(metaclass=Singleton): default_yaml_file = PROJECT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): - self._configs = {} - self._init_with_config_files_and_env(self._configs, yaml_file) + self._init_with_config_files_and_env(yaml_file) logger.info("Config loading done.") + self._update() + + def _update(self): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") @@ -96,12 +100,10 @@ def __init__(self, yaml_file=default_yaml_file): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") - self.git_repo = None - self.src_workspace = None - def _init_with_config_files_and_env(self, configs: dict, yaml_file): + def _init_with_config_files_and_env(self, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" - configs.update(os.environ) + configs = dict(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): @@ -112,11 +114,13 @@ def _init_with_config_files_and_env(self, configs: dict, yaml_file): yaml_data = yaml.safe_load(file) if not yaml_data: continue - os.environ.update({k: v for k, v in yaml_data.items() if isinstance(v, str)}) configs.update(yaml_data) + OPTIONS.set(configs) - def _get(self, *args, **kwargs): - return self._configs.get(*args, **kwargs) + @staticmethod + def _get(*args, **kwargs): + m = OPTIONS.get() + return m.get(*args, **kwargs) def get(self, key, *args, **kwargs): """Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found""" @@ -125,5 +129,33 @@ def get(self, key, *args, **kwargs): raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value + def __setattr__(self, name: str, value: Any) -> None: + OPTIONS.get()[name] = value + + def __getattr__(self, name: str) -> Any: + m = OPTIONS.get() + return m.get(name) + + def set_context(self, options: dict): + """Update current config""" + if not options: + return + opts = deepcopy(OPTIONS.get()) + opts.update(options) + OPTIONS.set(opts) + self._update() + + @property + def options(self): + """Return all key-values""" + return OPTIONS.get() + + def new_environ(self): + """Return a new os.environ object""" + env = os.environ.copy() + m = self.options + env.update({k: v for k, v in m.items() if isinstance(v, str)}) + return env + CONFIG = Config() diff --git a/metagpt/const.py b/metagpt/const.py index ce06655f1..9278a5d0e 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -8,6 +8,7 @@ common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. """ +import contextvars from pathlib import Path @@ -27,6 +28,8 @@ def get_project_root(): current_path = parent_path +OPTIONS = contextvars.ContextVar("OPTIONS") + PROJECT_ROOT = get_project_root() DATA_PATH = PROJECT_ROOT / "data" WORKSPACE_ROOT = PROJECT_ROOT / "workspace" From b794e5d73dc47722996508af4824b1e5496869a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 28 Nov 2023 16:22:40 +0800 Subject: [PATCH 175/232] feat: fix memory.add --- metagpt/roles/engineer.py | 4 ++-- metagpt/roles/role.py | 4 +++- metagpt/roles/sk_agent.py | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index c0e1b8a10..ffd96849b 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -168,7 +168,7 @@ async def _act_sp(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=self._rc.todo, - send_to="Edward", + send_to="Edward", # name of QaEngineer ) return msg @@ -209,7 +209,7 @@ async def _act_sp_precision(self) -> Message: content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=self._rc.todo, - send_to="Edward", + send_to="Edward", # name of QaEngineer ) return msg diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index e1f43ef3a..dbf800c03 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -217,6 +217,7 @@ async def _act(self) -> Message: ) else: msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) + self._rc.memory.add(msg) return msg @@ -227,7 +228,8 @@ async def _observe(self) -> int: # Store the read messages in your own memory to prevent duplicate processing. self._rc.memory.add_batch(news) # Filter out messages of interest. - self._rc.news = [n for n in news if n.cause_by in self._rc.watch] + old_messages = self._rc.memory.get() + self._rc.news = [n for n in news if n.cause_by in self._rc.watch and n not in old_messages] # Design Rules: # If you need to further categorize Message objects, you can do so using the Message.set_meta function. diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index 15b18dd3e..2443b8b58 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -74,5 +74,4 @@ async def _act(self) -> Message: msg = Message(content=result, role=self.profile, cause_by=self._rc.todo) self._rc.memory.add(msg) - self.publish_message(msg) return msg From 9745dd12f6bdb27bc5bbc16401605c3f9bbe688c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 28 Nov 2023 16:28:53 +0800 Subject: [PATCH 176/232] feat: fix memory.add --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index dbf800c03..62c8b7708 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -226,9 +226,9 @@ async def _observe(self) -> int: # Read unprocessed messages from the msg buffer. news = self._rc.msg_buffer.pop_all() # Store the read messages in your own memory to prevent duplicate processing. + old_messages = self._rc.memory.get() self._rc.memory.add_batch(news) # Filter out messages of interest. - old_messages = self._rc.memory.get() self._rc.news = [n for n in news if n.cause_by in self._rc.watch and n not in old_messages] # Design Rules: From 49f0b5e9f140c5d1a9a1b88289a0488a465601b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 28 Nov 2023 16:41:32 +0800 Subject: [PATCH 177/232] feat: fix memory.add --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f7de58d5a..1c9da7e6c 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -214,7 +214,7 @@ async def _act(self) -> Message: ) else: msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) - self._rc.memory.add(msg) + self._rc.memory.add(msg) return msg From f2de34fdad26a31def47a136f7ed7f73fa58ddf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 28 Nov 2023 16:42:15 +0800 Subject: [PATCH 178/232] feat: fix memory.add --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 62c8b7708..424a28c6f 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -217,7 +217,7 @@ async def _act(self) -> Message: ) else: msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) - self._rc.memory.add(msg) + self._rc.memory.add(msg) return msg From db9e900838b2c8eac4558fc858ce4a4e0cee6e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 09:52:26 +0800 Subject: [PATCH 179/232] feat: merge geekan:cli-etc --- metagpt/llm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/llm.py b/metagpt/llm.py index 14bbad1b4..a35ba354b 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,10 +8,13 @@ from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude +from metagpt.provider.human_provider import HumanProvider from metagpt.provider.openai_api import OpenAIGPTAPI from metagpt.provider.spark_api import SparkAPI from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +_ = HumanProvider() + def LLM() -> "BaseGPTAPI": """initialize different LLM instance according to the key field existence""" From f564bb540a97cc76229a46d51d0cde21980770ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 09:52:55 +0800 Subject: [PATCH 180/232] feat: merge geekan:cli-etc --- metagpt/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index a35ba354b..d8d06c0a1 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -13,7 +13,7 @@ from metagpt.provider.spark_api import SparkAPI from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI -_ = HumanProvider() +_ = HumanProvider() # Avoid pre-commit error def LLM() -> "BaseGPTAPI": From eff1cb7dc1ef842fc55d9118386052eadb98cf93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 10:14:04 +0800 Subject: [PATCH 181/232] feat: Add 'id' to 'Message' according to Section 2.2.3.1.1 of RFC 135. --- metagpt/schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 9e5854997..d1174799a 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -10,12 +10,14 @@ 1. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. 2. Encapsulate the common key-values set to pydantic structures to standardize and unify parameter passing between actions. + 3. Add `id` to `Message` according to Section 2.2.3.1.1 of RFC 135. """ from __future__ import annotations import asyncio import json import os.path +import uuid from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError from pathlib import Path @@ -86,6 +88,7 @@ class Documents(BaseModel): class Message(BaseModel): """list[: ]""" + id: str # According to Section 2.2.3.1.1 of RFC 135 content: str instruct_content: BaseModel = Field(default=None) role: str = "user" # system / user / assistant @@ -113,6 +116,7 @@ def __init__( :param role: Message meta info tells who sent this message. """ super().__init__( + id=uuid.uuid4().hex, content=content, instruct_content=instruct_content, role=role, From 7b44fccf8d826aa881b2fdd1765343a9a7207c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 16:22:05 +0800 Subject: [PATCH 182/232] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 7 ++- metagpt/startup.py | 16 +++++- metagpt/team.py | 11 ++-- metagpt/utils/git_repository.py | 4 +- startup.py | 80 ---------------------------- 5 files changed, 26 insertions(+), 92 deletions(-) delete mode 100644 startup.py diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 30558c93f..fe954b79c 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -12,7 +12,7 @@ from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME, WORKSPACE_ROOT +from metagpt.const import DEFAULT_WORKSPACE_ROOT, DOCS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.schema import Document from metagpt.utils.file_repository import FileRepository from metagpt.utils.git_repository import GitRepository @@ -28,8 +28,11 @@ async def run(self, with_messages, **kwargs): return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) # Create and initialize the workspace folder, initialize the Git environment. + default_workspace_root = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT + default_project_name = CONFIG.project_name or FileRepository.new_filename() + default_workdir = Path(default_workspace_root) / default_project_name CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else default_workdir CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. diff --git a/metagpt/startup.py b/metagpt/startup.py index 35b9b8b66..de348780b 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -4,6 +4,8 @@ import typer +from metagpt.config import CONFIG + app = typer.Typer() @@ -17,6 +19,10 @@ def startup( implement: bool = typer.Option(True, help="Enable or disable code implementation."), project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."), inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."), + project_path: str = typer.Option( + help="Specify the directory path of the old version project to fulfill the " "incremental requirements." + ), + reqa_file: str = typer.Option(help="Specify the source file name for rewriting the quality test code."), ): """Run a startup. Be a boss.""" from metagpt.roles import ( @@ -28,6 +34,12 @@ def startup( ) from metagpt.team import Team + # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. + CONFIG.project_name = project_name + CONFIG.inc = inc + CONFIG.project_path = project_path + CONFIG.reqa_file = reqa_file + company = Team() company.hire( [ @@ -44,9 +56,9 @@ def startup( company.hire([QaEngineer()]) company.invest(investment) - company.run_project(idea, project_name=project_name, inc=inc) + company.run_project(idea) asyncio.run(company.run(n_round=n_round)) if __name__ == "__main__": - startup(idea="Make a 2048 game.") + app() diff --git a/metagpt/team.py b/metagpt/team.py index e252935c4..92f379c97 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -11,6 +11,7 @@ from metagpt.actions import UserRequirement from metagpt.config import CONFIG +from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role @@ -45,16 +46,14 @@ def _check_balance(self): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}") - def run_project(self, idea, send_to: str = "", project_name: str = "", inc: bool = False): + def run_project(self, idea, send_to: str = ""): """Start a project from publishing user requirement.""" self.idea = idea - # If user set project_name, then use it. - if project_name: - path = CONFIG.workspace_path / project_name - self.env.load_existing_repo(path, inc=inc) # Human requirement. - self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) + self.env.publish_message( + Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL) + ) def _save(self): logger.info(self.json(ensure_ascii=False)) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index b8e35199b..b1cfe1ed2 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -16,7 +16,7 @@ from git.repo import Repo from git.repo.fun import is_git_dir -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile from metagpt.utils.file_repository import FileRepository @@ -201,7 +201,7 @@ def rename_root(self, new_dir_name): if __name__ == "__main__": - path = WORKSPACE_ROOT / "git" + path = DEFAULT_WORKSPACE_ROOT / "git" path.mkdir(exist_ok=True, parents=True) repo = GitRepository() diff --git a/startup.py b/startup.py deleted file mode 100644 index 1a59e7fa2..000000000 --- a/startup.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import asyncio - -import fire - -from metagpt.config import CONFIG -from metagpt.roles import ( - Architect, - Engineer, - ProductManager, - ProjectManager, - QaEngineer, -) -from metagpt.software_company import SoftwareCompany - - -async def startup( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = False, - run_tests: bool = False, - implement: bool = True, -): - """Run a startup. Be a boss.""" - company = SoftwareCompany() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) - - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) - - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) - - -def main( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = True, - run_tests: bool = False, - implement: bool = True, - project_path: str = None, - reqa_file: str = None, -): - """ - We are a software startup comprised of AI. By investing in us, - you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." - :param investment: As an investor, you have the opportunity to contribute - a certain dollar amount to this AI company. - :param n_round: - :param code_review: Whether to use code review. - :param run_tests: Whether run unit tests. - :param implement: Whether to write codes. - :param project_path: The path of the old version project to improve. - :return: - """ - CONFIG.WORKDIR = project_path - CONFIG.REQA_FILENAME = reqa_file - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) - - -if __name__ == "__main__": - fire.Fire(main) From 94043a89f41fa5da81d1fc56e0a1866423ae87d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 20:12:03 +0800 Subject: [PATCH 183/232] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 8 ++------ metagpt/startup.py | 19 ++++++++++--------- metagpt/utils/file_repository.py | 4 ++-- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index fe954b79c..71c94d25a 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -8,8 +8,6 @@ RFC 135 2.2.3.5.1. """ -from pathlib import Path - from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import DEFAULT_WORKSPACE_ROOT, DOCS_FILE_REPO, REQUIREMENT_FILENAME @@ -28,11 +26,9 @@ async def run(self, with_messages, **kwargs): return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) # Create and initialize the workspace folder, initialize the Git environment. - default_workspace_root = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT - default_project_name = CONFIG.project_name or FileRepository.new_filename() - default_workdir = Path(default_workspace_root) / default_project_name + project_name = CONFIG.project_name or FileRepository.new_filename() + workdir = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else default_workdir CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. diff --git a/metagpt/startup.py b/metagpt/startup.py index de348780b..78f32d556 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -12,17 +12,18 @@ @app.command() def startup( idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"), - investment: float = typer.Option(3.0, help="Dollar amount to invest in the AI company."), - n_round: int = typer.Option(5, help="Number of rounds for the simulation."), - code_review: bool = typer.Option(True, help="Whether to use code review."), - run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), - implement: bool = typer.Option(True, help="Enable or disable code implementation."), - project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."), - inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."), + investment: float = typer.Option(default=3.0, help="Dollar amount to invest in the AI company."), + n_round: int = typer.Option(default=5, help="Number of rounds for the simulation."), + code_review: bool = typer.Option(default=True, help="Whether to use code review."), + run_tests: bool = typer.Option(default=False, help="Whether to enable QA for adding & running tests."), + implement: bool = typer.Option(default=True, help="Enable or disable code implementation."), + project_name: str = typer.Option(default="", help="Unique project name, such as 'game_2048'."), + inc: bool = typer.Option(default=False, help="Incremental mode. Use it to coop with existing repo."), project_path: str = typer.Option( - help="Specify the directory path of the old version project to fulfill the " "incremental requirements." + default="", + help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", ), - reqa_file: str = typer.Option(help="Specify the source file name for rewriting the quality test code."), + reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), ): """Run a startup. Be a boss.""" from metagpt.roles import ( diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 846bfcd0c..0815bf90a 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -54,7 +54,7 @@ async def save(self, filename: Path | str, content, dependencies: List[str] = No """ pathname = self.workdir / filename pathname.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(str(pathname), mode="wb") as writer: + async with aiofiles.open(str(pathname), mode="w") as writer: await writer.write(content) logger.info(f"save to: {str(pathname)}") @@ -98,7 +98,7 @@ async def get(self, filename: Path | str) -> Document | None: if not path_name.exists(): return None try: - async with aiofiles.open(str(path_name), mode="rb") as reader: + async with aiofiles.open(str(path_name), mode="r") as reader: doc.content = await reader.read() except FileNotFoundError as e: logger.info(f"open {str(path_name)} failed:{e}") From 09b6d2df8377883335696c50f21d8956518f75ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 20:26:26 +0800 Subject: [PATCH 184/232] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 431879c25..2b9c20047 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -281,11 +281,7 @@ async def _new_system_design(self, context, format=CONFIG.prompt_format): # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" # contain space, have to use setattr - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) + self._rename_project_name(system_design=system_design) await self._rename_workspace(system_design) # ======= # # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr @@ -303,17 +299,29 @@ async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr + self._rename_project_name(system_design=system_design) + system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) + return system_design_doc + + @staticmethod + def _rename_project_name(system_design): + if CONFIG.project_name: + setattr( + system_design.instruct_content, + "project_name", + CONFIG.project_name, + ) + return setattr( system_design.instruct_content, "project_name", system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), ) - system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) - return system_design_doc @staticmethod async def _rename_workspace(system_design): - if CONFIG.WORKDIR: # Updating on the old version has already been specified if it's valid. + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 return if isinstance(system_design, ActionOutput): From 6fa3deef00ad06c68c77d68e60b349e0d0137ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 20:58:41 +0800 Subject: [PATCH 185/232] feat: merge geekan:cli-etc --- metagpt/actions/write_code.py | 28 ++++++++++++---------------- metagpt/utils/git_repository.py | 1 + 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index e373b1127..0cd41c52f 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -31,22 +31,6 @@ Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ------ -# Context -{context} ------ - -## Code: {filename} Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. -8. Before using a variable, make sure you reference it first -9. Write out EVERY DETAIL, DON'T LEAVE TODO. - ----- # Design ```json @@ -68,6 +52,18 @@ {logs} ``` ----- + +## Code: {filename} Write code with triple quoto, based on the following list and context. +1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. +2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. +5. Think before writing: What should be implemented and provided in this document? +6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +7. Do not use public member functions that do not exist in your design. +8. Before using a variable, make sure you reference it first +9. Write out EVERY DETAIL, DON'T LEAVE TODO. + ## Format example ----- ## Code: {filename} diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index b1cfe1ed2..7c9ec645f 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -59,6 +59,7 @@ def open(self, local_path: Path, auto_init=False): :param local_path: The local path to the Git repository. :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. """ + local_path = Path(local_path) if self.is_git_dir(local_path): self._repository = Repo(local_path) return From 810768a3505bb06dc9ae76024073432c671269ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 21:34:29 +0800 Subject: [PATCH 186/232] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 2b9c20047..3e17239b0 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -172,7 +172,7 @@ class Game{ ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here From cd24931b65b4a6ca9e926174c0a86dbdc3b1856c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 22:06:40 +0800 Subject: [PATCH 187/232] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 3e17239b0..e31ea76a8 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -166,24 +166,22 @@ class Game{ {context} ----- -Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a state-of-the-art (SOTA) Python system compliant with PEP8. Additionally, the objective is to optimize the use of high-quality open-source tools. -Requirement: Fill in the following missing information based on the context, each section name is a key in json -Max Output: 8192 chars or 2048 tokens. Try to use them up. +Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately +ATTENTION: Output carefully referenced "Old Design" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, -and only output the json inside this tag, nothing else +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """ From e34b8bbf0bbb9d46757da1eb33883fd8b554df56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 22:13:10 +0800 Subject: [PATCH 188/232] feat: merge geekan:cli-etc --- metagpt/actions/write_prd.py | 47 ++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index ab216b7a0..c61684918 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -261,24 +261,41 @@ {old_prd} ----- Role: You are a professional product manager; The goal is to merge the newly added requirements into the existing PRD in order to design a concise, usable, and efficient product. +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design +ATTENTION: Output carefully referenced "Old PRD" in format. -## Original Requirements: Provide as Plain text, place the polished complete original requirements here - -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## YOU NEED TO FULFILL THE BELOW JSON DOC -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +{{ + "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. + "Original Requirements": "", # str, place the polished complete original requirements here + "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Search Information": "", + "Requirements": "", + "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. + "User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories + "Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses + # Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + "Competitive Quadrant Chart": "quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + Campaign A: [0.3, 0.6] + Campaign B: [0.45, 0.23] + Campaign C: [0.57, 0.69] + Campaign D: [0.78, 0.34] + Campaign E: [0.40, 0.34] + Campaign F: [0.35, 0.78]", + "Requirement Analysis": "", # Provide as Plain text. + "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards + "UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. + "Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it. +}} output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old PRD" format, and only output the json inside this tag, nothing else From 6e0fc042258c005a8842cff66923d6bb76aa0e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 22:25:47 +0800 Subject: [PATCH 189/232] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index e31ea76a8..7164cef26 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -182,6 +182,9 @@ class Game{ ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. ## Anything UNCLEAR: Provide as Plain text. Try to clarify it. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, +and only output the json inside this tag, nothing else """ From 4928a896ca8b357bdfbc6d08d3b72d86f4598995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 10:16:34 +0800 Subject: [PATCH 190/232] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 71c94d25a..b339d897d 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -7,6 +7,7 @@ @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ +import shutil from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG @@ -28,6 +29,8 @@ async def run(self, with_messages, **kwargs): # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() workdir = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name + if not CONFIG.inc and workdir.exists(): + shutil.rmtree(workdir) CONFIG.git_repo = GitRepository() CONFIG.git_repo.open(local_path=workdir, auto_init=True) From 6010ce70f651d514db06f0014f07d12ec2e7c354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 12:13:45 +0800 Subject: [PATCH 191/232] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 3 ++- metagpt/actions/write_prd.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b339d897d..3d202e762 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -8,6 +8,7 @@ RFC 135 2.2.3.5.1. """ import shutil +from pathlib import Path from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG @@ -28,7 +29,7 @@ async def run(self, with_messages, **kwargs): # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() - workdir = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name + workdir = Path(CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name) if not CONFIG.inc and workdir.exists(): shutil.rmtree(workdir) CONFIG.git_repo = GitRepository() diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index c61684918..3967a0578 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -260,7 +260,7 @@ ## Old PRD {old_prd} ----- -Role: You are a professional product manager; The goal is to merge the newly added requirements into the existing PRD in order to design a concise, usable, and efficient product. +Role: You are a professional product manager; The goal is to incorporate the newly added requirements from the "Original Requirements" into the existing Product Requirements Document (PRD) in the "Old PRD" in order to design a concise, usable, and efficient product. Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design ATTENTION: Output carefully referenced "Old PRD" in format. From 5351b50d1cf963eebe6473783eb1860bdf6266c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 12:47:48 +0800 Subject: [PATCH 192/232] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 62 ++--------------------------------- 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 7164cef26..c5787ba20 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -227,72 +227,12 @@ async def run(self, with_messages, format=CONFIG.prompt_format): # leaving room for global optimization in subsequent steps. return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - # ======= - # def recreate_workspace(self, workspace: Path): - # try: - # shutil.rmtree(workspace) - # except FileNotFoundError: - # pass # Folder does not exist, but we don't care - # workspace.mkdir(parents=True, exist_ok=True) - - # async def _save_prd(self, docs_path, resources_path, context): - # prd_file = docs_path / "prd.md" - # if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: - # quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - # await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") - # - # if context[-1].instruct_content: - # logger.info(f"Saving PRD to {prd_file}") - # prd_file.write_text(context[-1].instruct_content.json(ensure_ascii=False), encoding='utf-8') - - # async def _save_system_design(self, docs_path, resources_path, system_design): - # data_api_design = system_design.instruct_content.dict()[ - # "Data structures and interfaces" - # ] # CodeParser.parse_code(block="Data structures and interfaces", text=content) - # seq_flow = system_design.instruct_content.dict()[ - # "Program call flow" - # ] # CodeParser.parse_code(block="Program call flow", text=content) - # await mermaid_to_file(data_api_design, resources_path / "data_api_design") - # await mermaid_to_file(seq_flow, resources_path / "seq_flow") - # system_design_file = docs_path / "system_design.md" - # logger.info(f"Saving System Designs to {system_design_file}") - # system_design_file.write_text(system_design.instruct_content.json(ensure_ascii=False), encoding='utf-8') - - # async def _save(self, context, system_design): - # if isinstance(system_design, ActionOutput): - # project_name = system_design.instruct_content.dict()["project_name"] - # else: - # project_name = CodeParser.parse_str(block="project_name", text=system_design) - # workspace = CONFIG.workspace_path / project_name - # self.recreate_workspace(workspace) - # docs_path = workspace / "docs" - # resources_path = workspace / "resources" - # docs_path.mkdir(parents=True, exist_ok=True) - # resources_path.mkdir(parents=True, exist_ok=True) - # await self._save_prd(docs_path, resources_path, context) - # await self._save_system_design(docs_path, resources_path, system_design) - - # async def run(self, context, format=CONFIG.prompt_format): - async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) - # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr self._rename_project_name(system_design=system_design) await self._rename_workspace(system_design) - # ======= - # # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr - # # setattr( - # # system_design.instruct_content, - # # "project_name", - # # system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - # # ) - # await self._save(context, system_design) - # >>>>>>> feature/geekan_cli_etc return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): @@ -306,6 +246,8 @@ async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): @staticmethod def _rename_project_name(system_design): + # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" + # contain space, have to use setattr if CONFIG.project_name: setattr( system_design.instruct_content, From 17bf646539ec2851c791c3b9cccabd8fdbf1753d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 12:53:02 +0800 Subject: [PATCH 193/232] feat: merge geekan:cli-etc --- metagpt/actions/project_management.py | 15 --------- metagpt/actions/write_code.py | 23 -------------- metagpt/actions/write_code_review.py | 17 ----------- metagpt/const.py | 28 ----------------- metagpt/environment.py | 44 --------------------------- metagpt/roles/engineer.py | 27 ---------------- metagpt/roles/qa_engineer.py | 26 ---------------- 7 files changed, 180 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index db856e55b..3d59daeed 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -258,21 +258,6 @@ async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo return task_doc async def _run_new_tasks(self, context, format=CONFIG.prompt_format): - # ======= - # def _save(self, context, rsp): - # if context[-1].instruct_content: - # ws_name = context[-1].instruct_content.dict()["project_name"] - # else: - # ws_name = CodeParser.parse_str(block="project_name", text=context[-1].content) - # file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" - # file_path.write_text(rsp.instruct_content.json(ensure_ascii=False)) - # - # # Write requirements.txt - # requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" - # requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) - # - # async def run(self, context, format=CONFIG.prompt_format): - # >>>>>>> feature/geekan_cli_etc prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 0cd41c52f..59ccb49a5 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -79,29 +79,6 @@ class WriteCode(Action): def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) - # <<<<<<< HEAD - # ======= - # def _is_invalid(self, filename): - # return any(i in filename for i in ["mp3", "wav"]) - # - # def _save(self, context, filename, code): - # # logger.info(filename) - # # logger.info(code_rsp) - # if self._is_invalid(filename): - # return - # - # design = [i for i in context if i.cause_by == WriteDesign][0] - # - # ws_name = CodeParser.parse_str(block="project_name", text=design.content) - # ws_path = CONFIG.workspace_path / ws_name - # if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - # ws_path = ws_path / ws_name - # code_path = ws_path / filename - # code_path.parent.mkdir(parents=True, exist_ok=True) - # code_path.write_text(code) - # logger.info(f"Saving Code to {code_path}") - # - # >>>>>>> feature/geekan_cli_etc @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 6d405029a..364f6af57 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -103,23 +103,6 @@ async def write_code_review_and_rewrite(self, prompt): code = CodeParser.parse_code(block="", text=code_rsp) return result, code - # <<<<<<< HEAD - # async def run(self, *args, **kwargs) -> CodingContext: - # format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) - # context = "\n----------\n".join( - # [self.context.design_doc.content, self.context.task_doc.content, self.context.code_doc.content] - # ) - # prompt = PROMPT_TEMPLATE.format( - # context=context, - # code=self.context.code_doc.content, - # filename=self.context.code_doc.filename, - # format_example=format_example, - # ) - # logger.info(f"Code review {self.context.code_doc.filename}..") - # code = await self.write_code(prompt) - # self.context.code_doc.content = code - # return self.context - # ======= async def run(self, *args, **kwargs) -> CodingContext: iterative_code = self.context.code_doc.content k = CONFIG.code_review_k_times or 1 diff --git a/metagpt/const.py b/metagpt/const.py index 6e616e820..a646cea7a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -19,34 +19,6 @@ OPTIONS = contextvars.ContextVar("OPTIONS") -# <<<<<<< HEAD -# def get_project_root(): -# """Search upwards to find the project root directory.""" -# current_path = Path.cwd() -# while True: -# if ( -# (current_path / ".git").exists() -# or (current_path / ".project_root").exists() -# or (current_path / ".gitignore").exists() -# ): -# return current_path -# parent_path = current_path.parent -# if parent_path == current_path: -# raise Exception("Project root not found.") -# current_path = parent_path -# -# -# PROJECT_ROOT = get_project_root() -# DATA_PATH = PROJECT_ROOT / "data" -# WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -# PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" -# UT_PATH = PROJECT_ROOT / "data/ut" -# SWAGGER_PATH = UT_PATH / "files/api/" -# UT_PY_PATH = UT_PATH / "files/ut/" -# API_QUESTIONS_PATH = UT_PATH / "files/question/" -# YAPI_URL = "http://yapi.deepwisdomai.com/" -# TMP = PROJECT_ROOT / "tmp" -# ======= def get_metagpt_package_root(): """Get the root directory of the installed package.""" package_root = Path(metagpt.__file__).parent.parent diff --git a/metagpt/environment.py b/metagpt/environment.py index e8bdd25c7..02eb3d340 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -23,7 +23,6 @@ class Environment(BaseModel): - # <<<<<<< HEAD """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles @@ -32,17 +31,6 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) members: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug - # ======= - # """ - # Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - # """ - # - # roles: dict[str, Role] = Field(default_factory=dict) - # memory: Memory = Field(default_factory=Memory) # 已经私有化 - # history: str = Field(default='') - # repo: Repo = Field(default_factory=Repo) # 在CONFIG里 - # kv: dict = Field(default_factory=dict) # 在CONFIG里 - # >>>>>>> feature/geekan_cli_etc class Config: arbitrary_types_allowed = True @@ -83,38 +71,6 @@ def publish_message(self, message: Message) -> bool: return True - # # Replaced by FileRepository.set_file - # def set_doc(self, content: str, filename: str): - # """向当前环境发布文档（包括代码）""" - # return self.repo.set(content, filename) - # - # # Replaced by FileRepository.get_file - # def get_doc(self, filename: str): - # return self.repo.get(filename) - # - # # Replaced by CONFIG.xx - # def set(self, k: str, v: str): - # self.kv[k] = v - # - # # Replaced by CONFIG.xx - # def get(self, k: str): - # return self.kv.get(k, None) - - # Replaced By 增量变更流程 - # def load_existing_repo(self, path: Path, inc: bool): - # self.repo = Repo.from_path(path) - # logger.info(self.repo.eda()) - # - # # Incremental mode: publish all docs to messages. Then roles can read the docs. - # if inc: - # docs = self.repo.get_text_documents() - # for doc in docs: - # msg = Message(content=doc.content) - # self.publish_message(msg) - # logger.info(f"Message from existing doc {doc.path}: {msg}") - # logger.info(f"Load {len(docs)} docs from existing repo.") - # raise NotImplementedError - async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 48262989c..78a7f3ba2 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -183,24 +183,6 @@ async def _act_sp_with_cr(self, review=False) -> Set[str]: msg = Message( content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode ) - # ======= - # context = [] - # msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - # for m in msg: - # context.append(m.content) - # context_str = "\n----------\n".join(context) - # # Write code - # code = await WriteCode().run(context=context_str, filename=todo) - # # Code review - # if self.use_code_review: - # # try: - # rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - # code = rewrite_code - # # except Exception as e: - # # logger.error("code review failed!", e) - # file_path = self.write_file(todo, code) - # msg = Message(content=code, role=self.profile, cause_by=WriteCode) - # >>>>>>> feature/geekan_cli_etc self._rc.memory.add(msg) changed_files.add(coding_context.code_doc.filename) @@ -273,15 +255,6 @@ async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_r coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) return coding_doc - # ======= - # async def _act(self) -> Message: - # """Determines the mode of action based on whether code review is used.""" - # logger.info(f"{self._setting}: ready to WriteCode") - # if self.use_code_review: - # return await self._act_sp_with_cr() - # return await self._act_sp() - # >>>>>>> feature/geekan_cli_etc - async def _new_code_actions(self): # Prepare file repos src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index fd2dcc786..ac5a280bb 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -44,32 +44,6 @@ def __init__( self.test_round = 0 self.test_round_allowed = test_round_allowed - # <<<<<<< HEAD - # ======= - # @classmethod - # def parse_workspace(cls, system_design_msg: Message) -> str: - # if system_design_msg.instruct_content: - # return system_design_msg.instruct_content.dict().get("project_name") - # return CodeParser.parse_str(block="project_name", text=system_design_msg.content) - # - # def get_workspace(self, return_proj_dir=True) -> Path: - # msg = self._rc.memory.get_by_action(WriteDesign)[-1] - # if not msg: - # return CONFIG.workspace_path / "src" - # workspace = self.parse_workspace(msg) - # # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. - # if return_proj_dir: - # return CONFIG.workspace_path / workspace - # # development codes directory: workspace/{package_name}/{package_name} - # return CONFIG.workspace_path / workspace / workspace - # - # def write_file(self, filename: str, code: str): - # workspace = self.get_workspace() / "tests" - # file = workspace / filename - # file.parent.mkdir(parents=True, exist_ok=True) - # file.write_text(code) - # - # >>>>>>> feature/geekan_cli_etc async def _write_test(self, message: Message) -> None: changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) From 6146d4dc7f352bc417bc2c720061613c7373ad39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 13:04:29 +0800 Subject: [PATCH 194/232] feat: merge RFC 135 --- metagpt/roles/engineer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 2bb1f3ea2..d42835a1b 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -218,15 +218,15 @@ async def _new_summarize_actions(self): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) changed_src_files = src_file_repo.changed_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). - summerizations = {} + summarizations = {} for filename in changed_src_files: - depenencies = src_file_repo.get_dependency(filename=filename) - ctx = CodeSummarizeContext.loads(filenames=depenencies) - if ctx not in summerizations: - summerizations[ctx] = set() - srcs = summerizations.get(ctx) + dependencies = src_file_repo.get_dependency(filename=filename) + ctx = CodeSummarizeContext.loads(filenames=dependencies) + if ctx not in summarizations: + summarizations[ctx] = set() + srcs = summarizations.get(ctx) srcs.add(filename) - for ctx, filenames in summerizations.items(): + for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) if self.summarize_todos: From 6f3d1d6f5e6c6080bab47ad28184c698c5dd7913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 14:58:06 +0800 Subject: [PATCH 195/232] =?UTF-8?q?fixbug:=20=E5=A2=9E=E9=87=8F=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E6=97=B6=E9=9C=80=E6=B1=82=E6=B2=A1=E5=86=99=E5=85=A5?= =?UTF-8?q?docs/requirement.txt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/prepare_documents.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 30558c93f..8656de812 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -23,14 +23,11 @@ def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) async def run(self, with_messages, **kwargs): - if CONFIG.git_repo: - doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) - return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) - - # Create and initialize the workspace folder, initialize the Git environment. - CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() - CONFIG.git_repo.open(local_path=workdir, auto_init=True) + if not CONFIG.git_repo: + # Create and initialize the workspace folder, initialize the Git environment. + CONFIG.git_repo = GitRepository() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() + CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) From 269eee4643728c5e5f0f6ce3efffa854f88c8f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 19:20:53 +0800 Subject: [PATCH 196/232] fixbug: The assumption that messages in 'memory' have been processed has been revoked. --- metagpt/roles/product_manager.py | 3 +++ metagpt/roles/role.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index bc6771829..966115c0f 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -50,3 +50,6 @@ async def _think(self) -> None: else: self._set_state(0) return self._rc.todo + + async def _observe(self, ignore_memory=False) -> int: + return await super(ProductManager, self)._observe(ignore_memory=True) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1c9da7e6c..fe121ed1a 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -218,12 +218,12 @@ async def _act(self) -> Message: return msg - async def _observe(self) -> int: + async def _observe(self, ignore_memory=False) -> int: """Prepare new messages for processing from the message buffer and other sources.""" # Read unprocessed messages from the msg buffer. news = self._rc.msg_buffer.pop_all() # Store the read messages in your own memory to prevent duplicate processing. - old_messages = self._rc.memory.get() + old_messages = [] if ignore_memory else self._rc.memory.get() self._rc.memory.add_batch(news) # Filter out messages of interest. self._rc.news = [n for n in news if n.cause_by in self._rc.watch and n not in old_messages] From 5c149efee77c6a3c90382de7221f1370eab7d94c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 19:33:27 +0800 Subject: [PATCH 197/232] fixbug: The assumption that messages in 'memory' have been processed has been revoked. --- metagpt/roles/qa_engineer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 763ab6a3f..de09cc4f0 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -167,3 +167,6 @@ async def _act(self) -> Message: sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) + + async def _observe(self, ignore_memory=False) -> int: + return await super(QaEngineer, self)._observe(ignore_memory=True) From 053eac62bcd990b748a4ce4578345880d882b276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Dec 2023 13:01:43 +0800 Subject: [PATCH 198/232] feat: +annotation --- metagpt/roles/qa_engineer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index de09cc4f0..f2e011ffd 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -169,4 +169,6 @@ async def _act(self) -> Message: ) async def _observe(self, ignore_memory=False) -> int: + # This role has events that trigger and execute themselves based on conditions, and cannot rely on the + # content of memory to activate. return await super(QaEngineer, self)._observe(ignore_memory=True) From 4845dafb94966a502f153a8e5d223b19f60be2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 10:31:02 +0800 Subject: [PATCH 199/232] feat: +log --- tests/metagpt/test_gpt.py | 8 ++++++++ tests/metagpt/test_llm.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/tests/metagpt/test_gpt.py b/tests/metagpt/test_gpt.py index 285e8134c..431858d4c 100644 --- a/tests/metagpt/test_gpt.py +++ b/tests/metagpt/test_gpt.py @@ -15,6 +15,7 @@ class TestGPT: def test_llm_api_ask(self, llm_api): answer = llm_api.ask("hello chatgpt") + logger.info(answer) assert len(answer) > 0 # def test_gptapi_ask_batch(self, llm_api): @@ -23,16 +24,19 @@ def test_llm_api_ask(self, llm_api): def test_llm_api_ask_code(self, llm_api): answer = llm_api.ask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask(self, llm_api): answer = await llm_api.aask("hello chatgpt") + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask_code(self, llm_api): answer = await llm_api.aask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio @@ -41,3 +45,7 @@ async def test_llm_api_costs(self, llm_api): costs = llm_api.get_costs() logger.info(costs) assert costs.total_cost > 0 + + +# if __name__ == "__main__": +# pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 03341212b..49969a2af 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -32,3 +32,6 @@ async def test_llm_acompletion(llm): assert len(await llm.acompletion(hello_msg)) > 0 assert len(await llm.acompletion_batch([hello_msg])) > 0 assert len(await llm.acompletion_batch_text([hello_msg])) > 0 + +# if __name__ == "__main__": +# pytest.main([__file__, "-s"]) From 37703253a3d9a46cae573f1c23e44a5e4b342b59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 23:04:07 +0800 Subject: [PATCH 200/232] feat: +SummarizeCode, refactor project_name --- metagpt/actions/design_api.py | 61 ++---- metagpt/actions/prepare_documents.py | 2 +- metagpt/actions/project_management.py | 10 +- metagpt/actions/summarize_code.py | 9 +- metagpt/actions/write_code.py | 20 +- metagpt/actions/write_code_review.py | 3 +- metagpt/actions/write_prd.py | 48 ++++- metagpt/actions/write_test.py | 7 +- metagpt/const.py | 3 + metagpt/provider/base_gpt_api.py | 4 +- metagpt/roles/engineer.py | 132 +++++++++--- metagpt/roles/qa_engineer.py | 12 +- metagpt/roles/role.py | 3 +- metagpt/schema.py | 20 +- metagpt/startup.py | 11 +- metagpt/utils/dependency_file.py | 5 +- metagpt/utils/file_repository.py | 33 +++ metagpt/utils/git_repository.py | 38 +++- tests/conftest.py | 16 ++ tests/metagpt/actions/mock.py | 2 +- tests/metagpt/actions/test_debug_error.py | 90 ++++---- tests/metagpt/actions/test_design_api.py | 26 +-- .../metagpt/actions/test_prepare_documents.py | 30 +++ tests/metagpt/actions/test_run_code.py | 62 +++--- tests/metagpt/actions/test_summarize_code.py | 195 ++++++++++++++++++ tests/metagpt/actions/test_write_code.py | 17 +- .../metagpt/actions/test_write_code_review.py | 12 +- tests/metagpt/actions/test_write_prd.py | 7 +- tests/metagpt/actions/test_write_test.py | 22 +- tests/metagpt/roles/mock.py | 2 +- tests/metagpt/utils/test_file_repository.py | 4 + 31 files changed, 663 insertions(+), 243 deletions(-) create mode 100644 tests/metagpt/actions/test_prepare_documents.py create mode 100644 tests/metagpt/actions/test_summarize_code.py diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index c5787ba20..eb73ed94f 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ import json from pathlib import Path @@ -23,7 +24,6 @@ ) from metagpt.logs import logger from metagpt.schema import Document, Documents -from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -43,7 +43,7 @@ ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text. ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -58,15 +58,15 @@ """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Implementation approach": "We will ...", - "project_name": "snake_game", + "Project name": "{project_name}", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram - class Game{ + class Game{{ +int score - } + }} ... Game "1" -- "1" Food: has ', @@ -77,7 +77,7 @@ class Game{ G->>M: end game ', "Anything UNCLEAR": "The requirement is clear to me." -} +}} [/CONTENT] """, }, @@ -96,7 +96,7 @@ class Game{ ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text. ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -112,9 +112,9 @@ class Game{ ## Implementation approach We will ... -## project_name +## Project name ```python -"snake_game" +"{project_name}" ``` ## File list @@ -151,7 +151,7 @@ class Game{ OUTPUT_MAPPING = { "Implementation approach": (str, ...), - "project_name": (str, ...), + "Project name": (str, ...), "File list": (List[str], ...), "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), @@ -173,7 +173,7 @@ class Game{ ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text "{project_name}". ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -229,50 +229,21 @@ async def run(self, with_messages, format=CONFIG.prompt_format): async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) + format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - self._rename_project_name(system_design=system_design) - await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + prompt = MERGE_PROMPT.format( + old_design=system_design_doc.content, context=prd_doc.content, project_name=CONFIG.project_name + ) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr - self._rename_project_name(system_design=system_design) system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) return system_design_doc - @staticmethod - def _rename_project_name(system_design): - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr - if CONFIG.project_name: - setattr( - system_design.instruct_content, - "project_name", - CONFIG.project_name, - ) - return - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) - - @staticmethod - async def _rename_workspace(system_design): - if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to - # Section 2.2.3.10 of RFC 135 - return - - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["project_name"] - else: - ws_name = CodeParser.parse_str(block="project_name", text=system_design) - CONFIG.git_repo.rename_root(ws_name) - async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: prd = await prds_file_repo.get(filename) old_system_design_doc = await system_design_file_repo.get(filename) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b751dc970..4a2082a07 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -3,7 +3,7 @@ """ @Time : 2023/11/20 @Author : mashenquan -@File : git_repository.py +@File : prepare_documents.py @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 3d59daeed..95da0d65a 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -183,6 +183,10 @@ ## Old Tasks {old_tasks} ----- + +## Format example +{format_example} +----- Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -201,7 +205,7 @@ ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format, and only output the json inside this tag, nothing else """ @@ -264,7 +268,9 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format): return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + _, format_example = get_template(templates, format) + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, + format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) task_doc.content = rsp.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index 88a37536b..d10cd6c55 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -3,12 +3,15 @@ """ @Author : alexanderwu @File : summarize_code.py +@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. """ +from pathlib import Path from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.utils.file_repository import FileRepository @@ -95,8 +98,10 @@ async def summarize_code(self, prompt): return code_rsp async def run(self): - design_doc = await FileRepository.get_file(self.context.design_filename) - task_doc = await FileRepository.get_file(self.context.task_filename) + design_pathname = Path(self.context.design_filename) + design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO) + task_pathname = Path(self.context.task_filename) + task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO) src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) code_blocks = [] for filename in self.context.codes_filenames: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 59ccb49a5..9b20843c7 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -15,13 +15,13 @@ RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ - from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action -from metagpt.const import TEST_OUTPUTS_FILE_REPO +from metagpt.config import CONFIG +from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger -from metagpt.schema import CodingContext, RunCodeResult +from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository @@ -50,6 +50,8 @@ # Debug logs ```text {logs} + +{summary_log} ``` ----- @@ -90,18 +92,26 @@ async def run(self, *args, **kwargs) -> CodingContext: test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) + summary_doc = None + if coding_context.design_doc.filename: + summary_doc = await FileRepository.get_file( + filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO + ) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr prompt = PROMPT_TEMPLATE.format( design=coding_context.design_doc.content, - tasks=coding_context.task_doc.content, - code=coding_context.code_doc.content, + tasks=coding_context.task_doc.content if coding_context.task_doc else "", + code=coding_context.code_doc.content if coding_context.code_doc else "", logs=logs, filename=self.context.filename, + summary_log=summary_doc.content if summary_doc else "", ) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) + if not coding_context.code_doc: + coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace) coding_context.code_doc.content = code return coding_context diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 364f6af57..f7c6845d2 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -108,10 +108,11 @@ async def run(self, *args, **kwargs) -> CodingContext: k = CONFIG.code_review_k_times or 1 for i in range(k): format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) + task_content = self.context.task_doc.content if self.context.task_doc else "" context = "\n----------\n".join( [ "```text\n" + self.context.design_doc.content + "```\n", - "```text\n" + self.context.task_doc.content + "```\n", + "```text\n" + task_content + "```\n", "```python\n" + self.context.code_doc.content + "```\n", ] ) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 3967a0578..530a22def 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -8,6 +8,7 @@ 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ from __future__ import annotations @@ -27,6 +28,7 @@ ) from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -53,7 +55,7 @@ {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -85,9 +87,10 @@ """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Language": "", "Original Requirements": "", + "Project Name": "{project_name}", "Search Information": "", "Requirements": "", "Product Goals": [], @@ -111,7 +114,7 @@ "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], "UI Design draft": "", "Anything UNCLEAR": "", -} +}} [/CONTENT] """, }, @@ -228,6 +231,7 @@ OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), + "Project Name": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), "Competitive Analysis": (List[str], ...), @@ -270,7 +274,7 @@ {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -320,6 +324,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"REWRITE PRD:{prd_doc.filename}") # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( @@ -327,6 +332,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"NEW PRD:{prd_doc.filename}") # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the # 'publish' message to transition the workflow to the next stage. This design allows room for global # optimization in subsequent steps. @@ -343,32 +349,36 @@ async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, # logger.info(format) prompt_template, format_example = get_template(templates, format) + project_name = CONFIG.project_name if CONFIG.project_name else "" + format_example = format_example.format(project_name=project_name) # logger.info(prompt_template) # logger.info(format_example) prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example + requirements=requirements, search_information=info, format_example=format_example, project_name=project_name ) # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + await self._rename_workspace(prd) return prd async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: - m = json.loads(old_prd_doc.content) - if m.get("Original Requirements") == new_requirement_doc.content: - # There have been no changes in the requirements, so they are considered unrelated. - return False prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) res = await self._aask(prompt=prompt) - logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") if "YES" in res: return True return False async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + prompt = MERGE_PROMPT.format( + requirements=new_requirement_doc.content, old_prd=prd_doc.content, project_name=CONFIG.project_name + ) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) prd_doc.content = prd.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(prd) return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: @@ -404,3 +414,19 @@ async def _save_competitive_analysis(prd_doc): @staticmethod async def _save_pdf(prd_doc): await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) + + @staticmethod + async def _rename_workspace(prd): + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + return + + if not CONFIG.project_name: + if isinstance(prd, ActionOutput): + ws_name = prd.instruct_content.dict()["Project Name"] + else: + ws_name = CodeParser.parse_str(block="Project Name", text=prd) + CONFIG.project_name = ws_name + CONFIG.git_repo.rename_root(CONFIG.project_name) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 7cbb42e1d..65673807f 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -9,8 +9,9 @@ """ from metagpt.actions.action import Action from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO from metagpt.logs import logger -from metagpt.schema import TestingContext +from metagpt.schema import Document, TestingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -52,6 +53,10 @@ async def write_code(self, prompt): return code async def run(self, *args, **kwargs) -> TestingContext: + if not self.context.test_doc: + self.context.test_doc = Document( + filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO + ) prompt = PROMPT_TEMPLATE.format( code_to_test=self.context.code_doc.content, test_file_name=self.context.test_doc.filename, diff --git a/metagpt/const.py b/metagpt/const.py index a646cea7a..bd735a5e1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/12/5. Add directories for code summarization.. """ import contextvars import os @@ -87,5 +88,7 @@ def get_metagpt_root(): TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" +CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 565ae94f7..6c1dc8338 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,7 +38,7 @@ def ask(self, msg: str) -> str: rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, stream=True) -> str: if system_msgs: message = ( self._system_msgs(system_msgs) + [self._user_msg(msg)] @@ -49,7 +49,7 @@ async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: message = ( [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] ) - rsp = await self.acompletion_text(message, stream=True) + rsp = await self.acompletion_text(message, stream=stream) logger.debug(message) # logger.debug(rsp) return rsp diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d42835a1b..9f8eb6482 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -13,17 +13,25 @@ @Modified By: mashenquan, 2023-11-27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from __future__ import annotations import json +from collections import defaultdict from pathlib import Path from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.const import ( + CODE_SUMMARIES_FILE_REPO, + CODE_SUMMARIES_PDF_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, +) from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( @@ -33,6 +41,16 @@ Documents, Message, ) +from metagpt.utils.common import any_to_str, any_to_str_set + +IS_PASS_PROMPT = """ +{context} + +---- +Does the above log indicate anything that needs to be done? +If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format; +otherwise, answer 'YES' in JSON format. +""" class Engineer(Role): @@ -60,7 +78,7 @@ def __init__( """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -105,39 +123,88 @@ async def _act(self) -> Message | None: if self._rc.todo is None: return None if isinstance(self._rc.todo, WriteCode): - changed_files = await self._act_sp_with_cr(review=self.use_code_review) - # Unit tests only. - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: - changed_files.add(CONFIG.REQA_FILENAME) - return Message( - content="\n".join(changed_files), - role=self.profile, - cause_by=WriteCodeReview if self.use_code_review else WriteCode, - send_to="Edward", # The name of QaEngineer - ) + return await self._act_write_code() if isinstance(self._rc.todo, SummarizeCode): - summaries = [] - for todo in self.summarize_todos: - summary = await todo.run() - summaries.append(summary.json(ensure_ascii=False)) + return await self._act_summarize() + return None + + async def _act_write_code(self): + changed_files = await self._act_sp_with_cr(review=self.use_code_review) + return Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=self, + sent_from=self, + ) + + async def _act_summarize(self): + code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) + code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) + tasks = [] + src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir) + for todo in self.summarize_todos: + summary = await todo.run() + summary_filename = Path(todo.context.design_filename).with_suffix(".md").name + dependencies = {todo.context.design_filename, todo.context.task_filename} + for filename in todo.context.codes_filenames: + rpath = src_relative_path / filename + dependencies.add(str(rpath)) + await code_summaries_pdf_file_repo.save( + filename=summary_filename, content=summary, dependencies=dependencies + ) + is_pass, reason = await self._is_pass(summary) + if not is_pass: + todo.context.reason = reason + tasks.append(todo.context.dict()) + await code_summaries_file_repo.save( + filename=Path(todo.context.design_filename).name, + content=todo.context.json(), + dependencies=dependencies, + ) + else: + await code_summaries_file_repo.delete(filename=Path(todo.context.design_filename).name) + + logger.info(f"--max-auto-summarize-code={CONFIG.max_auto_summarize_code}") + if not tasks or CONFIG.max_auto_summarize_code == 0: return Message( - content="\n".join(summaries), + content="", role=self.profile, cause_by=SummarizeCode, - send_to=MESSAGE_ROUTE_TO_NONE, + sent_from=self, + send_to="Edward", # The name of QaEngineer ) - return None + # The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. + # This parameter is used for debugging the workflow. + CONFIG.max_auto_summarize_code -= 1 if CONFIG.max_auto_summarize_code > 0 else 0 + return Message( + content=json.dumps(tasks), role=self.profile, cause_by=SummarizeCode, send_to=self, sent_from=self + ) + + async def _is_pass(self, summary) -> (str, str): + rsp = await self._llm.aask(msg=IS_PASS_PROMPT.format(context=summary), stream=False) + logger.info(rsp) + if "YES" in rsp: + return True, rsp + return False, rsp async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - if not self.code_todos: + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) + if not self._rc.news: + return None + msg = self._rc.news[0] + if msg.cause_by in write_code_filters: + logger.info(f"TODO WriteCode:{msg.json()}") await self._new_code_actions() - elif not self.summarize_todos: + return self._rc.todo + if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): + logger.info(f"TODO SummarizeCode:{msg.json()}") await self._new_summarize_actions() - else: - return None - return self._rc.todo # For agent store + return self._rc.todo + return None @staticmethod async def _new_coding_context( @@ -151,9 +218,9 @@ async def _new_coding_context( design_doc = None for i in dependencies: if str(i.parent) == TASK_FILE_REPO: - task_doc = task_file_repo.get(i.filename) + task_doc = await task_file_repo.get(i.name) elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: - design_doc = design_file_repo.get(i.filename) + design_doc = await design_file_repo.get(i.name) context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) return context @@ -216,16 +283,13 @@ async def _new_code_actions(self): async def _new_summarize_actions(self): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + src_files = src_file_repo.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). - summarizations = {} - for filename in changed_src_files: - dependencies = src_file_repo.get_dependency(filename=filename) + summarizations = defaultdict(list) + for filename in src_files: + dependencies = await src_file_repo.get_dependency(filename=filename) ctx = CodeSummarizeContext.loads(filenames=dependencies) - if ctx not in summarizations: - summarizations[ctx] = set() - srcs = summarizations.get(ctx) - srcs.add(filename) + summarizations[ctx].append(filename) for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 41a3213dc..15a01b9e9 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,10 +11,13 @@ WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message to using file references. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest # from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_TO_NONE, @@ -40,13 +43,16 @@ def __init__( self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed async def _write_test(self, message: Message) -> None: - changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_files = set(src_file_repo.changed_files.keys()) + # Unit tests only. + if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files: + changed_files.add(CONFIG.reqa_file) tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) for filename in changed_files: # write tests @@ -146,7 +152,7 @@ async def _act(self) -> Message: ) return result_msg - code_filters = any_to_str_set({WriteCode, WriteCodeReview}) + code_filters = any_to_str_set({SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e99cc1ff..2651be7eb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -284,9 +284,10 @@ async def _act(self) -> Message: instruct_content=response.instruct_content, role=self.profile, cause_by=self._rc.todo, + sent_from=self, ) else: - msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index d1174799a..a8c1b7726 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -48,9 +48,9 @@ class Document(BaseModel): Represents a document. """ - root_path: str - filename: str - content: Optional[str] = None + root_path: str = "" + filename: str = "" + content: str = "" def get_meta(self) -> Document: """Get metadata of the document. @@ -260,8 +260,8 @@ def load(self, v) -> "MessageQueue": class CodingContext(BaseModel): filename: str design_doc: Document - task_doc: Document - code_doc: Document + task_doc: Optional[Document] + code_doc: Optional[Document] @staticmethod def loads(val: str) -> CodingContext | None: @@ -275,7 +275,7 @@ def loads(val: str) -> CodingContext | None: class TestingContext(BaseModel): filename: str code_doc: Document - test_doc: Document + test_doc: Optional[Document] @staticmethod def loads(val: str) -> TestingContext | None: @@ -324,10 +324,11 @@ def loads(val: str) -> RunCodeResult | None: class CodeSummarizeContext(BaseModel): design_filename: str = "" task_filename: str = "" - codes_filenames: Set[str] = Field(default_factory=set) + codes_filenames: List[str] = Field(default_factory=list) + reason: str = "" @staticmethod - def loads(filenames: Set) -> CodeSummarizeContext: + def loads(filenames: List) -> CodeSummarizeContext: ctx = CodeSummarizeContext() for filename in filenames: if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): @@ -337,3 +338,6 @@ def loads(filenames: Set) -> CodeSummarizeContext: ctx.task_filename = str(filename) continue return ctx + + def __hash__(self): + return hash((self.design_filename, self.task_filename)) diff --git a/metagpt/startup.py b/metagpt/startup.py index 78f32d556..f930c386b 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import asyncio +from pathlib import Path import typer @@ -24,6 +25,10 @@ def startup( help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", ), reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), + max_auto_summarize_code: int = typer.Option( + default=-1, + help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.", + ), ): """Run a startup. Be a boss.""" from metagpt.roles import ( @@ -36,10 +41,14 @@ def startup( from metagpt.team import Team # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. + CONFIG.project_path = project_path + if project_path: + inc = True + project_name = project_name or Path(project_path).name CONFIG.project_name = project_name CONFIG.inc = inc - CONFIG.project_path = project_path CONFIG.reqa_file = reqa_file + CONFIG.max_auto_summarize_code = max_auto_summarize_code company = Team() company.hire( diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index 653e07ef9..e8347d567 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -14,6 +14,7 @@ import aiofiles +from metagpt.config import CONFIG from metagpt.logs import logger @@ -81,7 +82,7 @@ async def update(self, filename: Path | str, dependencies: Set[Path | str], pers if persist: await self.save() - async def get(self, filename: Path | str, persist=False): + async def get(self, filename: Path | str, persist=True): """Get dependencies for a file asynchronously. :param filename: The filename or path. @@ -91,7 +92,7 @@ async def get(self, filename: Path | str, persist=False): if persist: await self.load() - root = self._filename.parent + root = CONFIG.git_repo.workdir try: key = Path(filename).relative_to(root) except ValueError: diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 0815bf90a..2cace7232 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -151,6 +151,17 @@ def changed_files(self) -> Dict[str, str]: relative_files[str(rf)] = ct return relative_files + @property + def all_files(self) -> List: + """Get a dictionary of all files in the repository. + + The dictionary includes file paths relative to the current FileRepository. + + :return: A dictionary where keys are file paths and values are file information. + :rtype: List + """ + return self._git_repo.get_files(relative_path=self._relative_path) + def get_change_dir_files(self, dir: Path | str) -> List: """Get the files in a directory that have changed. @@ -259,3 +270,25 @@ async def save_as( """ file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies) + + async def delete(self, filename: Path | str): + """Delete a file from the file repository. + + This method deletes a file from the file repository based on the provided filename. + + :param filename: The name or path of the file to be deleted. + :type filename: Path or str + """ + pathname = self.workdir / filename + if not pathname.exists(): + return + pathname.unlink(missing_ok=True) + + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(filename=pathname, dependencies=None) + logger.info(f"remove dependency key: {str(pathname)}") + + @staticmethod + async def delete_file(filename: Path | str, relative_path: Path | str = "."): + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + await file_repo.delete(filename=filename) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 7c9ec645f..9a9ed0fce 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,10 +8,11 @@ """ from __future__ import annotations +import os import shutil from enum import Enum from pathlib import Path -from typing import Dict +from typing import Dict, List from git.repo import Repo from git.repo.fun import is_git_dir @@ -196,10 +197,43 @@ def rename_root(self, new_dir_name): if new_path.exists(): logger.info(f"Delete directory {str(new_path)}") shutil.rmtree(new_path) - self.workdir.rename(new_path) + os.rename(src=str(self.workdir), dst=str(new_path)) # self.workdir.rename(new_path) logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) + def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None) -> List: + """Retrieve a list of files in the specified relative path. + + The method returns a list of file paths relative to the current FileRepository. + + :param relative_path: The relative path within the repository. + :type relative_path: Path or str + :param root_relative_path: The root relative path within the repository. + :type root_relative_path: Path or str + :return: A list of file paths in the specified directory. + :rtype: List[str] + """ + try: + relative_path = Path(relative_path).relative_to(self.workdir) + except ValueError: + relative_path = Path(relative_path) + + if not root_relative_path: + root_relative_path = Path(self.workdir) / relative_path + files = [] + try: + directory_path = Path(self.workdir) / relative_path + for file_path in directory_path.iterdir(): + if file_path.is_file(): + rpath = file_path.relative_to(root_relative_path) + files.append(str(rpath)) + else: + subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path) + files.extend(subfolder_files) + except Exception as e: + logger.error(f"Error: {e}") + return files + if __name__ == "__main__": path = DEFAULT_WORKSPACE_ROOT / "git" diff --git a/tests/conftest.py b/tests/conftest.py index d2ac8304f..8e4422700 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,8 +12,11 @@ import pytest +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI +from metagpt.utils.git_repository import GitRepository class Context: @@ -68,3 +71,16 @@ async def handle_client(reader, writer): server = asyncio.get_event_loop().run_until_complete(asyncio.start_server(handle_client, "127.0.0.1", 0)) return "http://{}:{}".format(*server.sockets[0].getsockname()) + + +# init & dispose git repo +@pytest.fixture(scope="session", autouse=True) +def setup_and_teardown_git_repo(request): + CONFIG.git_repo = GitRepository(local_path=DEFAULT_WORKSPACE_ROOT / "unittest") + + # Destroy git repo at the end of the test session. + def fin(): + CONFIG.git_repo.delete_repository() + + # Register the function for destroying the environment. + request.addfinalizer(fin) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index c48913755..f6602a82b 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -90,7 +90,7 @@ For testing, we can use the PyTest framework. This is a mature full-featured Python testing tool that helps you write better programs. -## project_name: +## Project Name: ```python "adventure_game" ``` diff --git a/tests/metagpt/actions/test_debug_error.py b/tests/metagpt/actions/test_debug_error.py index 2393d2cc9..8289fe41b 100644 --- a/tests/metagpt/actions/test_debug_error.py +++ b/tests/metagpt/actions/test_debug_error.py @@ -4,17 +4,19 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_debug_error.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ +import uuid + import pytest from metagpt.actions.debug_error import DebugError +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO +from metagpt.schema import RunCodeContext, RunCodeResult +from metagpt.utils.file_repository import FileRepository -EXAMPLE_MSG_CONTENT = ''' ---- -## Development Code File Name -player.py -## Development Code -```python +CODE_CONTENT = ''' from typing import List from deck import Deck from card import Card @@ -58,12 +60,9 @@ def calculate_score(self) -> int: if self.score > 21 and any(card.rank == 'A' for card in self.hand): self.score -= 10 return self.score +''' -``` -## Test File Name -test_player.py -## Test Code -```python +TEST_CONTENT = """ import unittest from blackjack_game.player import Player from blackjack_game.deck import Deck @@ -114,42 +113,41 @@ def test_player_calculate_score_with_multiple_aces(self): if __name__ == '__main__': unittest.main() -``` -## Running Command -python tests/test_player.py -## Running Output -standard output: ; -standard errors: ..F.. -====================================================================== -FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "tests/test_player.py", line 46, in test_player_calculate_score_with_multiple_aces - self.assertEqual(player.score, 12) -AssertionError: 22 != 12 - ----------------------------------------------------------------------- -Ran 5 tests in 0.007s - -FAILED (failures=1) -; -## instruction: -The error is in the development code, specifically in the calculate_score method of the Player class. The method is not correctly handling the case where there are multiple Aces in the player's hand. The current implementation only subtracts 10 from the score once if the score is over 21 and there's an Ace in the hand. However, in the case of multiple Aces, it should subtract 10 for each Ace until the score is 21 or less. -## File To Rewrite: -player.py -## Status: -FAIL -## Send To: -Engineer ---- -''' +""" @pytest.mark.asyncio async def test_debug_error(): - debug_error = DebugError("debug_error") - - file_name, rewritten_code = await debug_error.run(context=EXAMPLE_MSG_CONTENT) - - assert "class Player" in rewritten_code # rewrite the same class - assert "while self.score > 21" in rewritten_code # a key logic to rewrite to (original one is "if self.score > 12") + CONFIG.src_workspace = CONFIG.git_repo.workdir / uuid.uuid4().hex + ctx = RunCodeContext( + code_filename="player.py", + test_filename="test_player.py", + command=["python", "tests/test_player.py"], + output_filename="output.log", + ) + + await FileRepository.save_file(filename=ctx.code_filename, content=CODE_CONTENT, relative_path=CONFIG.src_workspace) + await FileRepository.save_file(filename=ctx.test_filename, content=TEST_CONTENT, relative_path=TEST_CODES_FILE_REPO) + output_data = RunCodeResult( + stdout=";", + stderr="", + summary="======================================================================\n" + "FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer)\n" + "----------------------------------------------------------------------\n" + "Traceback (most recent call last):\n" + ' File "tests/test_player.py", line 46, in test_player_calculate_score_' + "with_multiple_aces\n" + " self.assertEqual(player.score, 12)\nAssertionError: 22 != 12\n\n" + "----------------------------------------------------------------------\n" + "Ran 5 tests in 0.007s\n\nFAILED (failures=1)\n;\n", + ) + await FileRepository.save_file( + filename=ctx.output_filename, content=output_data.json(), relative_path=TEST_OUTPUTS_FILE_REPO + ) + debug_error = DebugError(context=ctx) + + rsp = await debug_error.run() + + assert "class Player" in rsp # rewrite the same class + # a key logic to rewrite to (original one is "if self.score > 12") + assert "while self.score > 21" in rsp diff --git a/tests/metagpt/actions/test_design_api.py b/tests/metagpt/actions/test_design_api.py index 0add8fb74..e90707d1a 100644 --- a/tests/metagpt/actions/test_design_api.py +++ b/tests/metagpt/actions/test_design_api.py @@ -4,33 +4,27 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : test_design_api.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.design_api import WriteDesign +from metagpt.const import PRDS_FILE_REPO from metagpt.logs import logger from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository from tests.metagpt.actions.mock import PRD_SAMPLE @pytest.mark.asyncio async def test_design_api(): - prd = "我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。" + inputs = ["我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", PRD_SAMPLE] + for prd in inputs: + await FileRepository.save_file("new_prd.txt", content=prd, relative_path=PRDS_FILE_REPO) - design_api = WriteDesign("design_api") + design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) + result = await design_api.run([Message(content=prd, instruct_content=None)]) + logger.info(result) - assert result - - -@pytest.mark.asyncio -async def test_design_api_calculator(): - prd = PRD_SAMPLE - - design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) - - assert result + assert result diff --git a/tests/metagpt/actions/test_prepare_documents.py b/tests/metagpt/actions/test_prepare_documents.py new file mode 100644 index 000000000..31c8bcb80 --- /dev/null +++ b/tests/metagpt/actions/test_prepare_documents.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/6 +@Author : mashenquan +@File : test_prepare_documents.py +@Desc: Unit test for prepare_documents.py +""" +import pytest + +from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository + + +@pytest.mark.asyncio +async def test_prepare_documents(): + msg = Message(content="New user requirements balabala...") + + if CONFIG.git_repo: + CONFIG.git_repo.delete_repository() + CONFIG.git_repo = None + + await PrepareDocuments().run(with_messages=[msg]) + assert CONFIG.git_repo + doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) + assert doc + assert doc.content == msg.content diff --git a/tests/metagpt/actions/test_run_code.py b/tests/metagpt/actions/test_run_code.py index 1e451cb14..888418974 100644 --- a/tests/metagpt/actions/test_run_code.py +++ b/tests/metagpt/actions/test_run_code.py @@ -4,10 +4,12 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_run_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.run_code import RunCode +from metagpt.schema import RunCodeContext @pytest.mark.asyncio @@ -35,37 +37,29 @@ async def test_run_script(): @pytest.mark.asyncio async def test_run(): - action = RunCode() - result = await action.run(mode="text", code="print('Hello, World')") - assert "PASS" in result - - result = await action.run( - mode="script", - code="echo 'Hello World'", - code_file_name="", - test_code="", - test_file_name="", - command=["echo", "Hello World"], - working_directory=".", - additional_python_paths=[], - ) - assert "PASS" in result - - -@pytest.mark.asyncio -async def test_run_failure(): - action = RunCode() - result = await action.run(mode="text", code="result = 1 / 0") - assert "FAIL" in result - - result = await action.run( - mode="script", - code='python -c "print(1/0)"', - code_file_name="", - test_code="", - test_file_name="", - command=["python", "-c", "print(1/0)"], - working_directory=".", - additional_python_paths=[], - ) - assert "FAIL" in result + inputs = [ + (RunCodeContext(mode="text", code_filename="a.txt", code="print('Hello, World')"), "PASS"), + ( + RunCodeContext( + mode="script", + code_filename="a.sh", + code="echo 'Hello World'", + command=["echo", "Hello World"], + working_directory=".", + ), + "PASS", + ), + ( + RunCodeContext( + mode="script", + code_filename="a.py", + code='python -c "print(1/0)"', + command=["python", "-c", "print(1/0)"], + working_directory=".", + ), + "FAIL", + ), + ] + for ctx, result in inputs: + rsp = await RunCode(context=ctx).run() + assert result in rsp.summary diff --git a/tests/metagpt/actions/test_summarize_code.py b/tests/metagpt/actions/test_summarize_code.py new file mode 100644 index 000000000..7ecb67afd --- /dev/null +++ b/tests/metagpt/actions/test_summarize_code.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 17:46 +@Author : mashenquan +@File : test_summarize_code.py +@Modifiled By: mashenquan, 2023-12-6. Unit test for summarize_code.py +""" +import pytest + +from metagpt.actions.summarize_code import SummarizeCode +from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.logs import logger +from metagpt.schema import CodeSummarizeContext +from metagpt.utils.file_repository import FileRepository + +DESIGN_CONTENT = """ +{"Implementation approach": "To develop this snake game, we will use the Python language and choose the Pygame library. Pygame is an open-source Python module collection specifically designed for writing video games. It provides functionalities such as displaying images and playing sounds, making it suitable for creating intuitive and responsive user interfaces. We will ensure efficient game logic to prevent any delays during gameplay. The scoring system will be simple, with the snake gaining points for each food it eats. We will use Pygame's event handling system to implement pause and resume functionality, as well as high-score tracking. The difficulty will increase by speeding up the snake's movement. In the initial version, we will focus on single-player mode and consider adding multiplayer mode and customizable skins in future updates. Based on the new requirement, we will also add a moving obstacle that appears randomly. If the snake eats this obstacle, the game will end. If the snake does not eat the obstacle, it will disappear after 5 seconds. For this, we need to add mechanisms for obstacle generation, movement, and disappearance in the game logic.", "Project_name": "snake_game", "File list": ["main.py", "game.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "constants.py", "assets/styles.css", "assets/index.html"], "Data structures and interfaces": "```mermaid\n classDiagram\n class Game{\n +int score\n +int speed\n +bool game_over\n +bool paused\n +Snake snake\n +Food food\n +Obstacle obstacle\n +Scoreboard scoreboard\n +start_game() void\n +pause_game() void\n +resume_game() void\n +end_game() void\n +increase_difficulty() void\n +update() void\n +render() void\n Game()\n }\n class Snake{\n +list body_parts\n +str direction\n +bool grow\n +move() void\n +grow() void\n +check_collision() bool\n Snake()\n }\n class Food{\n +tuple position\n +spawn() void\n Food()\n }\n class Obstacle{\n +tuple position\n +int lifetime\n +bool active\n +spawn() void\n +move() void\n +check_collision() bool\n +disappear() void\n Obstacle()\n }\n class Scoreboard{\n +int high_score\n +update_score(int) void\n +reset_score() void\n +load_high_score() void\n +save_high_score() void\n Scoreboard()\n }\n class Constants{\n }\n Game \"1\" -- \"1\" Snake: has\n Game \"1\" -- \"1\" Food: has\n Game \"1\" -- \"1\" Obstacle: has\n Game \"1\" -- \"1\" Scoreboard: has\n ```", "Program call flow": "```sequenceDiagram\n participant M as Main\n participant G as Game\n participant S as Snake\n participant F as Food\n participant O as Obstacle\n participant SB as Scoreboard\n M->>G: start_game()\n loop game loop\n G->>S: move()\n G->>S: check_collision()\n G->>F: spawn()\n G->>O: spawn()\n G->>O: move()\n G->>O: check_collision()\n G->>O: disappear()\n G->>SB: update_score(score)\n G->>G: update()\n G->>G: render()\n alt if paused\n M->>G: pause_game()\n M->>G: resume_game()\n end\n alt if game_over\n G->>M: end_game()\n end\n end\n```", "Anything UNCLEAR": "There is no need for further clarification as the requirements are already clear."} +""" + +TASK_CONTENT = """ +{"Required Python third-party packages": ["pygame==2.0.1"], "Required Other language third-party packages": ["No third-party packages required for other languages."], "Full API spec": "\n openapi: 3.0.0\n info:\n title: Snake Game API\n version: \"1.0.0\"\n paths:\n /start:\n get:\n summary: Start the game\n responses:\n '200':\n description: Game started successfully\n /pause:\n get:\n summary: Pause the game\n responses:\n '200':\n description: Game paused successfully\n /resume:\n get:\n summary: Resume the game\n responses:\n '200':\n description: Game resumed successfully\n /end:\n get:\n summary: End the game\n responses:\n '200':\n description: Game ended successfully\n /score:\n get:\n summary: Get the current score\n responses:\n '200':\n description: Current score retrieved successfully\n /highscore:\n get:\n summary: Get the high score\n responses:\n '200':\n description: High score retrieved successfully\n components: {}\n ", "Logic Analysis": [["constants.py", "Contains all the constant values like screen size, colors, game speeds, etc. This should be implemented first as it provides the base values for other components."], ["snake.py", "Contains the Snake class with methods for movement, growth, and collision detection. It is dependent on constants.py for configuration values."], ["food.py", "Contains the Food class responsible for spawning food items on the screen. It is dependent on constants.py for configuration values."], ["obstacle.py", "Contains the Obstacle class with methods for spawning, moving, and disappearing of obstacles, as well as collision detection with the snake. It is dependent on constants.py for configuration values."], ["scoreboard.py", "Contains the Scoreboard class for updating, resetting, loading, and saving high scores. It may use constants.py for configuration values and depends on the game's scoring logic."], ["game.py", "Contains the main Game class which includes the game loop and methods for starting, pausing, resuming, and ending the game. It is dependent on snake.py, food.py, obstacle.py, and scoreboard.py."], ["main.py", "The entry point of the game that initializes the game and starts the game loop. It is dependent on game.py."]], "Task list": ["constants.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "game.py", "main.py"], "Shared Knowledge": "\n 'constants.py' should contain all the necessary configurations for the game, such as screen dimensions, color definitions, and speed settings. These constants will be used across multiple files, ensuring consistency and ease of updates. Ensure that the Pygame library is initialized correctly in 'main.py' before starting the game loop. Also, make sure that the game's state is managed properly when pausing and resuming the game.\n ", "Anything UNCLEAR": "The interaction between the 'obstacle.py' and the game loop needs to be clearly defined to ensure obstacles appear and disappear correctly. The lifetime of the obstacle and its random movement should be implemented in a way that does not interfere with the game's performance."} +""" + +FOOD_PY = """ +## food.py +import random + +class Food: + def __init__(self): + self.position = (0, 0) + + def generate(self): + x = random.randint(0, 9) + y = random.randint(0, 9) + self.position = (x, y) + + def get_position(self): + return self.position + +""" + +GAME_PY = """ +## game.py +import pygame +from snake import Snake +from food import Food + +class Game: + def __init__(self): + self.score = 0 + self.level = 1 + self.snake = Snake() + self.food = Food() + + def start_game(self): + pygame.init() + self.initialize_game() + self.game_loop() + + def initialize_game(self): + self.score = 0 + self.level = 1 + self.snake.reset() + self.food.generate() + + def game_loop(self): + game_over = False + + while not game_over: + self.update() + self.draw() + self.handle_events() + self.check_collision() + self.increase_score() + self.increase_level() + + if self.snake.is_collision(): + game_over = True + self.game_over() + + def update(self): + self.snake.move() + + def draw(self): + self.snake.draw() + self.food.draw() + + def handle_events(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + quit() + elif event.type == pygame.KEYDOWN: + if event.key == pygame.K_UP: + self.snake.change_direction("UP") + elif event.key == pygame.K_DOWN: + self.snake.change_direction("DOWN") + elif event.key == pygame.K_LEFT: + self.snake.change_direction("LEFT") + elif event.key == pygame.K_RIGHT: + self.snake.change_direction("RIGHT") + + def check_collision(self): + if self.snake.get_head() == self.food.get_position(): + self.snake.grow() + self.food.generate() + + def increase_score(self): + self.score += 1 + + def increase_level(self): + if self.score % 10 == 0: + self.level += 1 + + def game_over(self): + print("Game Over") + self.initialize_game() + +""" + +MAIN_PY = """ +## main.py +import pygame +from game import Game + +def main(): + pygame.init() + game = Game() + game.start_game() + +if __name__ == "__main__": + main() + +""" + +SNAKE_PY = """ +## snake.py +import pygame + +class Snake: + def __init__(self): + self.body = [(0, 0)] + self.direction = (1, 0) + + def move(self): + head = self.body[0] + dx, dy = self.direction + new_head = (head[0] + dx, head[1] + dy) + self.body.insert(0, new_head) + self.body.pop() + + def change_direction(self, direction): + if direction == "UP": + self.direction = (0, -1) + elif direction == "DOWN": + self.direction = (0, 1) + elif direction == "LEFT": + self.direction = (-1, 0) + elif direction == "RIGHT": + self.direction = (1, 0) + + def grow(self): + tail = self.body[-1] + dx, dy = self.direction + new_tail = (tail[0] - dx, tail[1] - dy) + self.body.append(new_tail) + + def get_head(self): + return self.body[0] + + def get_body(self): + return self.body[1:] + +""" + + +@pytest.mark.asyncio +async def test_summarize_code(): + CONFIG.src_workspace = CONFIG.git_repo.workdir / "src" + await FileRepository.save_file(filename="1.json", relative_path=SYSTEM_DESIGN_FILE_REPO, content=DESIGN_CONTENT) + await FileRepository.save_file(filename="1.json", relative_path=TASK_FILE_REPO, content=TASK_CONTENT) + await FileRepository.save_file(filename="food.py", relative_path=CONFIG.src_workspace, content=FOOD_PY) + await FileRepository.save_file(filename="game.py", relative_path=CONFIG.src_workspace, content=GAME_PY) + await FileRepository.save_file(filename="main.py", relative_path=CONFIG.src_workspace, content=MAIN_PY) + await FileRepository.save_file(filename="snake.py", relative_path=CONFIG.src_workspace, content=SNAKE_PY) + + src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) + all_files = src_file_repo.all_files + ctx = CodeSummarizeContext(design_filename="1.json", task_filename="1.json", codes_filenames=all_files) + action = SummarizeCode(context=ctx) + rsp = await action.run() + assert rsp + logger.info(rsp) diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index eb5e3de91..54229089c 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,26 +4,31 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.write_code import WriteCode from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.schema import CodingContext, Document from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @pytest.mark.asyncio async def test_write_code(): - api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" - write_code = WriteCode("write_code") + context = CodingContext( + filename="task_filename.py", design_doc=Document(content="设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。") + ) + doc = Document(content=context.json()) + write_code = WriteCode(context=doc) - code = await write_code.run(api_design) - logger.info(code) + code = await write_code.run() + logger.info(code.json()) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 - assert "def add" in code - assert "return" in code + assert "def add" in code.code_doc.content + assert "return" in code.code_doc.content @pytest.mark.asyncio diff --git a/tests/metagpt/actions/test_write_code_review.py b/tests/metagpt/actions/test_write_code_review.py index 21bc563ec..e16eb7348 100644 --- a/tests/metagpt/actions/test_write_code_review.py +++ b/tests/metagpt/actions/test_write_code_review.py @@ -8,6 +8,8 @@ import pytest from metagpt.actions.write_code_review import WriteCodeReview +from metagpt.document import Document +from metagpt.schema import CodingContext @pytest.mark.asyncio @@ -16,13 +18,15 @@ async def test_write_code_review(capfd): def add(a, b): return a + """ - # write_code_review = WriteCodeReview("write_code_review") + context = CodingContext( + filename="math.py", design_doc=Document(content="编写一个从a加b的函数，返回a+b"), code_doc=Document(content=code) + ) - code = await WriteCodeReview().run(context="编写一个从a加b的函数，返回a+b", code=code, filename="math.py") + context = await WriteCodeReview(context=context).run() # 我们不能精确地预测生成的代码评审，但我们可以检查返回的是否为字符串 - assert isinstance(code, str) - assert len(code) > 0 + assert isinstance(context.code_doc.content, str) + assert len(context.code_doc.content) > 0 captured = capfd.readouterr() print(f"输出内容: {captured.out}") diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 8f8ef84f5..08be3cf75 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -9,19 +9,24 @@ import pytest from metagpt.actions import UserRequirement +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository @pytest.mark.asyncio async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" + await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=requirements, relative_path=DOCS_FILE_REPO) prd = await product_manager.run(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) # Assert the prd is not None or empty assert prd is not None - assert prd != "" + assert prd.content != "" + assert CONFIG.git_repo.new_file_repository(relative_path=PRDS_FILE_REPO).changed_files diff --git a/tests/metagpt/actions/test_write_test.py b/tests/metagpt/actions/test_write_test.py index e5acdff44..a3190fb0e 100644 --- a/tests/metagpt/actions/test_write_test.py +++ b/tests/metagpt/actions/test_write_test.py @@ -9,6 +9,7 @@ from metagpt.actions.write_test import WriteTest from metagpt.logs import logger +from metagpt.schema import Document, TestingContext @pytest.mark.asyncio @@ -24,22 +25,17 @@ def __init__(self, position: Tuple[int, int]): def generate(self, max_y: int, max_x: int): self.position = (random.randint(1, max_y - 1), random.randint(1, max_x - 1)) """ + context = TestingContext(filename="food.py", code_doc=Document(filename="food.py", content=code)) + write_test = WriteTest(context=context) - write_test = WriteTest() - - test_code = await write_test.run( - code_to_test=code, - test_file_name="test_food.py", - source_file_path="/some/dummy/path/cli_snake_game/cli_snake_game/food.py", - workspace="/some/dummy/path/cli_snake_game", - ) - logger.info(test_code) + context = await write_test.run() + logger.info(context.json()) # We cannot exactly predict the generated test cases, but we can check if it is a string and if it is not empty - assert isinstance(test_code, str) - assert "from cli_snake_game.food import Food" in test_code - assert "class TestFood(unittest.TestCase)" in test_code - assert "def test_generate" in test_code + assert isinstance(context.test_doc.content, str) + assert "from food import Food" in context.test_doc.content + assert "class TestFood(unittest.TestCase)" in context.test_doc.content + assert "def test_generate" in context.test_doc.content @pytest.mark.asyncio diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 5500b69f7..75f6b3b43 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -71,7 +71,7 @@ ``` ''' -SYSTEM_DESIGN = """## project_name +SYSTEM_DESIGN = """## Project name ```python "smart_search_engine" ``` diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py index a830b58aa..92e5204c5 100644 --- a/tests/metagpt/utils/test_file_repository.py +++ b/tests/metagpt/utils/test_file_repository.py @@ -43,6 +43,10 @@ async def test_file_repo(): assert {"a.txt"} == await file_repo.get_changed_dependency("b.txt") await file_repo.save("d/e.txt", "EEE") assert ["d/e.txt"] == file_repo.get_change_dir_files("d") + assert set(file_repo.all_files) == {"a.txt", "b.txt", "d/e.txt"} + await file_repo.delete("d/e.txt") + await file_repo.delete("d/e.txt") # delete twice + assert set(file_repo.all_files) == {"a.txt", "b.txt"} git_repo.delete_repository() From 2e0a847f63fc3c045dd296b8f84debee9cf6900f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Dec 2023 15:08:38 +0800 Subject: [PATCH 201/232] fixbug --- metagpt/utils/git_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 9a9ed0fce..8b53ce7d2 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -197,7 +197,7 @@ def rename_root(self, new_dir_name): if new_path.exists(): logger.info(f"Delete directory {str(new_path)}") shutil.rmtree(new_path) - os.rename(src=str(self.workdir), dst=str(new_path)) # self.workdir.rename(new_path) + shutil.move(src=str(self.workdir), dst=str(new_path)) # self.workdir.rename(new_path) logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) From ef633b7c26f809daaccf27a850df52cb1e349a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Dec 2023 17:09:21 +0800 Subject: [PATCH 202/232] fixbug: move dir --- metagpt/utils/git_repository.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 8b53ce7d2..5aec4509c 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -197,7 +197,10 @@ def rename_root(self, new_dir_name): if new_path.exists(): logger.info(f"Delete directory {str(new_path)}") shutil.rmtree(new_path) - shutil.move(src=str(self.workdir), dst=str(new_path)) # self.workdir.rename(new_path) + try: + shutil.move(src=str(self.workdir), dst=str(new_path)) + except Exception as e: + logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}") logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) From 9d84c8f047b60455bbf9c679af3cfe4cf1b11b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 23:04:07 +0800 Subject: [PATCH 203/232] feat: +SummarizeCode, refactor project_name --- metagpt/actions/design_api.py | 65 ++---- metagpt/actions/prepare_documents.py | 7 +- metagpt/actions/project_management.py | 10 +- metagpt/actions/summarize_code.py | 9 +- metagpt/actions/write_code.py | 20 +- metagpt/actions/write_code_review.py | 3 +- metagpt/actions/write_prd.py | 48 ++++- metagpt/actions/write_test.py | 7 +- metagpt/const.py | 3 + metagpt/provider/base_gpt_api.py | 4 +- metagpt/roles/engineer.py | 132 +++++++++--- metagpt/roles/qa_engineer.py | 12 +- metagpt/roles/role.py | 3 +- metagpt/schema.py | 20 +- metagpt/startup.py | 11 +- metagpt/utils/dependency_file.py | 5 +- metagpt/utils/file_repository.py | 33 +++ metagpt/utils/git_repository.py | 41 +++- tests/conftest.py | 16 ++ tests/metagpt/actions/mock.py | 2 +- tests/metagpt/actions/test_debug_error.py | 90 ++++---- tests/metagpt/actions/test_design_api.py | 26 +-- .../metagpt/actions/test_prepare_documents.py | 30 +++ tests/metagpt/actions/test_run_code.py | 62 +++--- tests/metagpt/actions/test_summarize_code.py | 195 ++++++++++++++++++ tests/metagpt/actions/test_write_code.py | 17 +- .../metagpt/actions/test_write_code_review.py | 12 +- tests/metagpt/actions/test_write_prd.py | 7 +- tests/metagpt/actions/test_write_test.py | 22 +- tests/metagpt/roles/mock.py | 2 +- tests/metagpt/utils/test_file_repository.py | 4 + 31 files changed, 672 insertions(+), 246 deletions(-) create mode 100644 tests/metagpt/actions/test_prepare_documents.py create mode 100644 tests/metagpt/actions/test_summarize_code.py diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index c5787ba20..557ebcbbd 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ import json from pathlib import Path @@ -23,7 +24,6 @@ ) from metagpt.logs import logger from metagpt.schema import Document, Documents -from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -43,7 +43,7 @@ ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text. ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -58,15 +58,15 @@ """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Implementation approach": "We will ...", - "project_name": "snake_game", + "Project name": "{project_name}", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram - class Game{ + class Game{{ +int score - } + }} ... Game "1" -- "1" Food: has ', @@ -77,7 +77,7 @@ class Game{ G->>M: end game ', "Anything UNCLEAR": "The requirement is clear to me." -} +}} [/CONTENT] """, }, @@ -96,7 +96,7 @@ class Game{ ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text. ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -112,9 +112,9 @@ class Game{ ## Implementation approach We will ... -## project_name +## Project name ```python -"snake_game" +"{project_name}" ``` ## File list @@ -151,7 +151,7 @@ class Game{ OUTPUT_MAPPING = { "Implementation approach": (str, ...), - "project_name": (str, ...), + "Project name": (str, ...), "File list": (List[str], ...), "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), @@ -173,7 +173,7 @@ class Game{ ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## Project name: Constant text "{project_name}". ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -229,50 +229,21 @@ async def run(self, with_messages, format=CONFIG.prompt_format): async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) + format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - self._rename_project_name(system_design=system_design) - await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + prompt = MERGE_PROMPT.format( + old_design=system_design_doc.content, context=prd_doc.content, project_name=CONFIG.project_name + ) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr - self._rename_project_name(system_design=system_design) system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) return system_design_doc - @staticmethod - def _rename_project_name(system_design): - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr - if CONFIG.project_name: - setattr( - system_design.instruct_content, - "project_name", - CONFIG.project_name, - ) - return - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) - - @staticmethod - async def _rename_workspace(system_design): - if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to - # Section 2.2.3.10 of RFC 135 - return - - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["project_name"] - else: - ws_name = CodeParser.parse_str(block="project_name", text=system_design) - CONFIG.git_repo.rename_root(ws_name) - async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: prd = await prds_file_repo.get(filename) old_system_design_doc = await system_design_file_repo.get(filename) @@ -296,10 +267,10 @@ async def _update_system_design(self, filename, prds_file_repo, system_design_fi @staticmethod async def _save_data_api_design(design_doc): m = json.loads(design_doc.content) - data_api_design = m.get("Data structures and interface definitions") + data_api_design = m.get("Data structures and interfaces") if not data_api_design: return - pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc.filename).with_suffix("") + pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b751dc970..05255dcc5 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -3,7 +3,7 @@ """ @Time : 2023/11/20 @Author : mashenquan -@File : git_repository.py +@File : prepare_documents.py @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ @@ -26,7 +26,10 @@ async def run(self, with_messages, **kwargs): if not CONFIG.git_repo: # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() - workdir = Path(CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name) + workdir = CONFIG.project_path + if not workdir and CONFIG.workspace: + workdir = Path(CONFIG.workspace) / project_name + workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name) if not CONFIG.inc and workdir.exists(): shutil.rmtree(workdir) CONFIG.git_repo = GitRepository() diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 3d59daeed..95da0d65a 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -183,6 +183,10 @@ ## Old Tasks {old_tasks} ----- + +## Format example +{format_example} +----- Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -201,7 +205,7 @@ ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format, and only output the json inside this tag, nothing else """ @@ -264,7 +268,9 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format): return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + _, format_example = get_template(templates, format) + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, + format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) task_doc.content = rsp.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index 88a37536b..d10cd6c55 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -3,12 +3,15 @@ """ @Author : alexanderwu @File : summarize_code.py +@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. """ +from pathlib import Path from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.utils.file_repository import FileRepository @@ -95,8 +98,10 @@ async def summarize_code(self, prompt): return code_rsp async def run(self): - design_doc = await FileRepository.get_file(self.context.design_filename) - task_doc = await FileRepository.get_file(self.context.task_filename) + design_pathname = Path(self.context.design_filename) + design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO) + task_pathname = Path(self.context.task_filename) + task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO) src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) code_blocks = [] for filename in self.context.codes_filenames: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 59ccb49a5..9b20843c7 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -15,13 +15,13 @@ RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ - from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action -from metagpt.const import TEST_OUTPUTS_FILE_REPO +from metagpt.config import CONFIG +from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger -from metagpt.schema import CodingContext, RunCodeResult +from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository @@ -50,6 +50,8 @@ # Debug logs ```text {logs} + +{summary_log} ``` ----- @@ -90,18 +92,26 @@ async def run(self, *args, **kwargs) -> CodingContext: test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) + summary_doc = None + if coding_context.design_doc.filename: + summary_doc = await FileRepository.get_file( + filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO + ) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr prompt = PROMPT_TEMPLATE.format( design=coding_context.design_doc.content, - tasks=coding_context.task_doc.content, - code=coding_context.code_doc.content, + tasks=coding_context.task_doc.content if coding_context.task_doc else "", + code=coding_context.code_doc.content if coding_context.code_doc else "", logs=logs, filename=self.context.filename, + summary_log=summary_doc.content if summary_doc else "", ) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) + if not coding_context.code_doc: + coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace) coding_context.code_doc.content = code return coding_context diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 364f6af57..f7c6845d2 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -108,10 +108,11 @@ async def run(self, *args, **kwargs) -> CodingContext: k = CONFIG.code_review_k_times or 1 for i in range(k): format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) + task_content = self.context.task_doc.content if self.context.task_doc else "" context = "\n----------\n".join( [ "```text\n" + self.context.design_doc.content + "```\n", - "```text\n" + self.context.task_doc.content + "```\n", + "```text\n" + task_content + "```\n", "```python\n" + self.context.code_doc.content + "```\n", ] ) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 3967a0578..530a22def 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -8,6 +8,7 @@ 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ from __future__ import annotations @@ -27,6 +28,7 @@ ) from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -53,7 +55,7 @@ {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -85,9 +87,10 @@ """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Language": "", "Original Requirements": "", + "Project Name": "{project_name}", "Search Information": "", "Requirements": "", "Product Goals": [], @@ -111,7 +114,7 @@ "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], "UI Design draft": "", "Anything UNCLEAR": "", -} +}} [/CONTENT] """, }, @@ -228,6 +231,7 @@ OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), + "Project Name": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), "Competitive Analysis": (List[str], ...), @@ -270,7 +274,7 @@ {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -320,6 +324,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"REWRITE PRD:{prd_doc.filename}") # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( @@ -327,6 +332,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"NEW PRD:{prd_doc.filename}") # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the # 'publish' message to transition the workflow to the next stage. This design allows room for global # optimization in subsequent steps. @@ -343,32 +349,36 @@ async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, # logger.info(format) prompt_template, format_example = get_template(templates, format) + project_name = CONFIG.project_name if CONFIG.project_name else "" + format_example = format_example.format(project_name=project_name) # logger.info(prompt_template) # logger.info(format_example) prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example + requirements=requirements, search_information=info, format_example=format_example, project_name=project_name ) # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + await self._rename_workspace(prd) return prd async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: - m = json.loads(old_prd_doc.content) - if m.get("Original Requirements") == new_requirement_doc.content: - # There have been no changes in the requirements, so they are considered unrelated. - return False prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) res = await self._aask(prompt=prompt) - logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") if "YES" in res: return True return False async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + prompt = MERGE_PROMPT.format( + requirements=new_requirement_doc.content, old_prd=prd_doc.content, project_name=CONFIG.project_name + ) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) prd_doc.content = prd.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(prd) return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: @@ -404,3 +414,19 @@ async def _save_competitive_analysis(prd_doc): @staticmethod async def _save_pdf(prd_doc): await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) + + @staticmethod + async def _rename_workspace(prd): + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + return + + if not CONFIG.project_name: + if isinstance(prd, ActionOutput): + ws_name = prd.instruct_content.dict()["Project Name"] + else: + ws_name = CodeParser.parse_str(block="Project Name", text=prd) + CONFIG.project_name = ws_name + CONFIG.git_repo.rename_root(CONFIG.project_name) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 7cbb42e1d..65673807f 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -9,8 +9,9 @@ """ from metagpt.actions.action import Action from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO from metagpt.logs import logger -from metagpt.schema import TestingContext +from metagpt.schema import Document, TestingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -52,6 +53,10 @@ async def write_code(self, prompt): return code async def run(self, *args, **kwargs) -> TestingContext: + if not self.context.test_doc: + self.context.test_doc = Document( + filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO + ) prompt = PROMPT_TEMPLATE.format( code_to_test=self.context.code_doc.content, test_file_name=self.context.test_doc.filename, diff --git a/metagpt/const.py b/metagpt/const.py index a646cea7a..bd735a5e1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/12/5. Add directories for code summarization.. """ import contextvars import os @@ -87,5 +88,7 @@ def get_metagpt_root(): TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" +CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 565ae94f7..6c1dc8338 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,7 +38,7 @@ def ask(self, msg: str) -> str: rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, stream=True) -> str: if system_msgs: message = ( self._system_msgs(system_msgs) + [self._user_msg(msg)] @@ -49,7 +49,7 @@ async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: message = ( [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] ) - rsp = await self.acompletion_text(message, stream=True) + rsp = await self.acompletion_text(message, stream=stream) logger.debug(message) # logger.debug(rsp) return rsp diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d42835a1b..9f8eb6482 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -13,17 +13,25 @@ @Modified By: mashenquan, 2023-11-27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from __future__ import annotations import json +from collections import defaultdict from pathlib import Path from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.const import ( + CODE_SUMMARIES_FILE_REPO, + CODE_SUMMARIES_PDF_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, +) from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( @@ -33,6 +41,16 @@ Documents, Message, ) +from metagpt.utils.common import any_to_str, any_to_str_set + +IS_PASS_PROMPT = """ +{context} + +---- +Does the above log indicate anything that needs to be done? +If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format; +otherwise, answer 'YES' in JSON format. +""" class Engineer(Role): @@ -60,7 +78,7 @@ def __init__( """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -105,39 +123,88 @@ async def _act(self) -> Message | None: if self._rc.todo is None: return None if isinstance(self._rc.todo, WriteCode): - changed_files = await self._act_sp_with_cr(review=self.use_code_review) - # Unit tests only. - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: - changed_files.add(CONFIG.REQA_FILENAME) - return Message( - content="\n".join(changed_files), - role=self.profile, - cause_by=WriteCodeReview if self.use_code_review else WriteCode, - send_to="Edward", # The name of QaEngineer - ) + return await self._act_write_code() if isinstance(self._rc.todo, SummarizeCode): - summaries = [] - for todo in self.summarize_todos: - summary = await todo.run() - summaries.append(summary.json(ensure_ascii=False)) + return await self._act_summarize() + return None + + async def _act_write_code(self): + changed_files = await self._act_sp_with_cr(review=self.use_code_review) + return Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=self, + sent_from=self, + ) + + async def _act_summarize(self): + code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) + code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) + tasks = [] + src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir) + for todo in self.summarize_todos: + summary = await todo.run() + summary_filename = Path(todo.context.design_filename).with_suffix(".md").name + dependencies = {todo.context.design_filename, todo.context.task_filename} + for filename in todo.context.codes_filenames: + rpath = src_relative_path / filename + dependencies.add(str(rpath)) + await code_summaries_pdf_file_repo.save( + filename=summary_filename, content=summary, dependencies=dependencies + ) + is_pass, reason = await self._is_pass(summary) + if not is_pass: + todo.context.reason = reason + tasks.append(todo.context.dict()) + await code_summaries_file_repo.save( + filename=Path(todo.context.design_filename).name, + content=todo.context.json(), + dependencies=dependencies, + ) + else: + await code_summaries_file_repo.delete(filename=Path(todo.context.design_filename).name) + + logger.info(f"--max-auto-summarize-code={CONFIG.max_auto_summarize_code}") + if not tasks or CONFIG.max_auto_summarize_code == 0: return Message( - content="\n".join(summaries), + content="", role=self.profile, cause_by=SummarizeCode, - send_to=MESSAGE_ROUTE_TO_NONE, + sent_from=self, + send_to="Edward", # The name of QaEngineer ) - return None + # The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. + # This parameter is used for debugging the workflow. + CONFIG.max_auto_summarize_code -= 1 if CONFIG.max_auto_summarize_code > 0 else 0 + return Message( + content=json.dumps(tasks), role=self.profile, cause_by=SummarizeCode, send_to=self, sent_from=self + ) + + async def _is_pass(self, summary) -> (str, str): + rsp = await self._llm.aask(msg=IS_PASS_PROMPT.format(context=summary), stream=False) + logger.info(rsp) + if "YES" in rsp: + return True, rsp + return False, rsp async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - if not self.code_todos: + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) + if not self._rc.news: + return None + msg = self._rc.news[0] + if msg.cause_by in write_code_filters: + logger.info(f"TODO WriteCode:{msg.json()}") await self._new_code_actions() - elif not self.summarize_todos: + return self._rc.todo + if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): + logger.info(f"TODO SummarizeCode:{msg.json()}") await self._new_summarize_actions() - else: - return None - return self._rc.todo # For agent store + return self._rc.todo + return None @staticmethod async def _new_coding_context( @@ -151,9 +218,9 @@ async def _new_coding_context( design_doc = None for i in dependencies: if str(i.parent) == TASK_FILE_REPO: - task_doc = task_file_repo.get(i.filename) + task_doc = await task_file_repo.get(i.name) elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: - design_doc = design_file_repo.get(i.filename) + design_doc = await design_file_repo.get(i.name) context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) return context @@ -216,16 +283,13 @@ async def _new_code_actions(self): async def _new_summarize_actions(self): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + src_files = src_file_repo.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). - summarizations = {} - for filename in changed_src_files: - dependencies = src_file_repo.get_dependency(filename=filename) + summarizations = defaultdict(list) + for filename in src_files: + dependencies = await src_file_repo.get_dependency(filename=filename) ctx = CodeSummarizeContext.loads(filenames=dependencies) - if ctx not in summarizations: - summarizations[ctx] = set() - srcs = summarizations.get(ctx) - srcs.add(filename) + summarizations[ctx].append(filename) for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 41a3213dc..15a01b9e9 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,10 +11,13 @@ WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message to using file references. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest # from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_TO_NONE, @@ -40,13 +43,16 @@ def __init__( self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed async def _write_test(self, message: Message) -> None: - changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_files = set(src_file_repo.changed_files.keys()) + # Unit tests only. + if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files: + changed_files.add(CONFIG.reqa_file) tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) for filename in changed_files: # write tests @@ -146,7 +152,7 @@ async def _act(self) -> Message: ) return result_msg - code_filters = any_to_str_set({WriteCode, WriteCodeReview}) + code_filters = any_to_str_set({SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e99cc1ff..2651be7eb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -284,9 +284,10 @@ async def _act(self) -> Message: instruct_content=response.instruct_content, role=self.profile, cause_by=self._rc.todo, + sent_from=self, ) else: - msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index d1174799a..a8c1b7726 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -48,9 +48,9 @@ class Document(BaseModel): Represents a document. """ - root_path: str - filename: str - content: Optional[str] = None + root_path: str = "" + filename: str = "" + content: str = "" def get_meta(self) -> Document: """Get metadata of the document. @@ -260,8 +260,8 @@ def load(self, v) -> "MessageQueue": class CodingContext(BaseModel): filename: str design_doc: Document - task_doc: Document - code_doc: Document + task_doc: Optional[Document] + code_doc: Optional[Document] @staticmethod def loads(val: str) -> CodingContext | None: @@ -275,7 +275,7 @@ def loads(val: str) -> CodingContext | None: class TestingContext(BaseModel): filename: str code_doc: Document - test_doc: Document + test_doc: Optional[Document] @staticmethod def loads(val: str) -> TestingContext | None: @@ -324,10 +324,11 @@ def loads(val: str) -> RunCodeResult | None: class CodeSummarizeContext(BaseModel): design_filename: str = "" task_filename: str = "" - codes_filenames: Set[str] = Field(default_factory=set) + codes_filenames: List[str] = Field(default_factory=list) + reason: str = "" @staticmethod - def loads(filenames: Set) -> CodeSummarizeContext: + def loads(filenames: List) -> CodeSummarizeContext: ctx = CodeSummarizeContext() for filename in filenames: if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): @@ -337,3 +338,6 @@ def loads(filenames: Set) -> CodeSummarizeContext: ctx.task_filename = str(filename) continue return ctx + + def __hash__(self): + return hash((self.design_filename, self.task_filename)) diff --git a/metagpt/startup.py b/metagpt/startup.py index 78f32d556..f930c386b 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import asyncio +from pathlib import Path import typer @@ -24,6 +25,10 @@ def startup( help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", ), reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), + max_auto_summarize_code: int = typer.Option( + default=-1, + help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.", + ), ): """Run a startup. Be a boss.""" from metagpt.roles import ( @@ -36,10 +41,14 @@ def startup( from metagpt.team import Team # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. + CONFIG.project_path = project_path + if project_path: + inc = True + project_name = project_name or Path(project_path).name CONFIG.project_name = project_name CONFIG.inc = inc - CONFIG.project_path = project_path CONFIG.reqa_file = reqa_file + CONFIG.max_auto_summarize_code = max_auto_summarize_code company = Team() company.hire( diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index 653e07ef9..e8347d567 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -14,6 +14,7 @@ import aiofiles +from metagpt.config import CONFIG from metagpt.logs import logger @@ -81,7 +82,7 @@ async def update(self, filename: Path | str, dependencies: Set[Path | str], pers if persist: await self.save() - async def get(self, filename: Path | str, persist=False): + async def get(self, filename: Path | str, persist=True): """Get dependencies for a file asynchronously. :param filename: The filename or path. @@ -91,7 +92,7 @@ async def get(self, filename: Path | str, persist=False): if persist: await self.load() - root = self._filename.parent + root = CONFIG.git_repo.workdir try: key = Path(filename).relative_to(root) except ValueError: diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 0815bf90a..2cace7232 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -151,6 +151,17 @@ def changed_files(self) -> Dict[str, str]: relative_files[str(rf)] = ct return relative_files + @property + def all_files(self) -> List: + """Get a dictionary of all files in the repository. + + The dictionary includes file paths relative to the current FileRepository. + + :return: A dictionary where keys are file paths and values are file information. + :rtype: List + """ + return self._git_repo.get_files(relative_path=self._relative_path) + def get_change_dir_files(self, dir: Path | str) -> List: """Get the files in a directory that have changed. @@ -259,3 +270,25 @@ async def save_as( """ file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies) + + async def delete(self, filename: Path | str): + """Delete a file from the file repository. + + This method deletes a file from the file repository based on the provided filename. + + :param filename: The name or path of the file to be deleted. + :type filename: Path or str + """ + pathname = self.workdir / filename + if not pathname.exists(): + return + pathname.unlink(missing_ok=True) + + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(filename=pathname, dependencies=None) + logger.info(f"remove dependency key: {str(pathname)}") + + @staticmethod + async def delete_file(filename: Path | str, relative_path: Path | str = "."): + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + await file_repo.delete(filename=filename) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 7c9ec645f..5aec4509c 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,10 +8,11 @@ """ from __future__ import annotations +import os import shutil from enum import Enum from pathlib import Path -from typing import Dict +from typing import Dict, List from git.repo import Repo from git.repo.fun import is_git_dir @@ -196,10 +197,46 @@ def rename_root(self, new_dir_name): if new_path.exists(): logger.info(f"Delete directory {str(new_path)}") shutil.rmtree(new_path) - self.workdir.rename(new_path) + try: + shutil.move(src=str(self.workdir), dst=str(new_path)) + except Exception as e: + logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}") logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) + def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None) -> List: + """Retrieve a list of files in the specified relative path. + + The method returns a list of file paths relative to the current FileRepository. + + :param relative_path: The relative path within the repository. + :type relative_path: Path or str + :param root_relative_path: The root relative path within the repository. + :type root_relative_path: Path or str + :return: A list of file paths in the specified directory. + :rtype: List[str] + """ + try: + relative_path = Path(relative_path).relative_to(self.workdir) + except ValueError: + relative_path = Path(relative_path) + + if not root_relative_path: + root_relative_path = Path(self.workdir) / relative_path + files = [] + try: + directory_path = Path(self.workdir) / relative_path + for file_path in directory_path.iterdir(): + if file_path.is_file(): + rpath = file_path.relative_to(root_relative_path) + files.append(str(rpath)) + else: + subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path) + files.extend(subfolder_files) + except Exception as e: + logger.error(f"Error: {e}") + return files + if __name__ == "__main__": path = DEFAULT_WORKSPACE_ROOT / "git" diff --git a/tests/conftest.py b/tests/conftest.py index d2ac8304f..8e4422700 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,8 +12,11 @@ import pytest +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI +from metagpt.utils.git_repository import GitRepository class Context: @@ -68,3 +71,16 @@ async def handle_client(reader, writer): server = asyncio.get_event_loop().run_until_complete(asyncio.start_server(handle_client, "127.0.0.1", 0)) return "http://{}:{}".format(*server.sockets[0].getsockname()) + + +# init & dispose git repo +@pytest.fixture(scope="session", autouse=True) +def setup_and_teardown_git_repo(request): + CONFIG.git_repo = GitRepository(local_path=DEFAULT_WORKSPACE_ROOT / "unittest") + + # Destroy git repo at the end of the test session. + def fin(): + CONFIG.git_repo.delete_repository() + + # Register the function for destroying the environment. + request.addfinalizer(fin) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index c48913755..f6602a82b 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -90,7 +90,7 @@ For testing, we can use the PyTest framework. This is a mature full-featured Python testing tool that helps you write better programs. -## project_name: +## Project Name: ```python "adventure_game" ``` diff --git a/tests/metagpt/actions/test_debug_error.py b/tests/metagpt/actions/test_debug_error.py index 2393d2cc9..8289fe41b 100644 --- a/tests/metagpt/actions/test_debug_error.py +++ b/tests/metagpt/actions/test_debug_error.py @@ -4,17 +4,19 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_debug_error.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ +import uuid + import pytest from metagpt.actions.debug_error import DebugError +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO +from metagpt.schema import RunCodeContext, RunCodeResult +from metagpt.utils.file_repository import FileRepository -EXAMPLE_MSG_CONTENT = ''' ---- -## Development Code File Name -player.py -## Development Code -```python +CODE_CONTENT = ''' from typing import List from deck import Deck from card import Card @@ -58,12 +60,9 @@ def calculate_score(self) -> int: if self.score > 21 and any(card.rank == 'A' for card in self.hand): self.score -= 10 return self.score +''' -``` -## Test File Name -test_player.py -## Test Code -```python +TEST_CONTENT = """ import unittest from blackjack_game.player import Player from blackjack_game.deck import Deck @@ -114,42 +113,41 @@ def test_player_calculate_score_with_multiple_aces(self): if __name__ == '__main__': unittest.main() -``` -## Running Command -python tests/test_player.py -## Running Output -standard output: ; -standard errors: ..F.. -====================================================================== -FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "tests/test_player.py", line 46, in test_player_calculate_score_with_multiple_aces - self.assertEqual(player.score, 12) -AssertionError: 22 != 12 - ----------------------------------------------------------------------- -Ran 5 tests in 0.007s - -FAILED (failures=1) -; -## instruction: -The error is in the development code, specifically in the calculate_score method of the Player class. The method is not correctly handling the case where there are multiple Aces in the player's hand. The current implementation only subtracts 10 from the score once if the score is over 21 and there's an Ace in the hand. However, in the case of multiple Aces, it should subtract 10 for each Ace until the score is 21 or less. -## File To Rewrite: -player.py -## Status: -FAIL -## Send To: -Engineer ---- -''' +""" @pytest.mark.asyncio async def test_debug_error(): - debug_error = DebugError("debug_error") - - file_name, rewritten_code = await debug_error.run(context=EXAMPLE_MSG_CONTENT) - - assert "class Player" in rewritten_code # rewrite the same class - assert "while self.score > 21" in rewritten_code # a key logic to rewrite to (original one is "if self.score > 12") + CONFIG.src_workspace = CONFIG.git_repo.workdir / uuid.uuid4().hex + ctx = RunCodeContext( + code_filename="player.py", + test_filename="test_player.py", + command=["python", "tests/test_player.py"], + output_filename="output.log", + ) + + await FileRepository.save_file(filename=ctx.code_filename, content=CODE_CONTENT, relative_path=CONFIG.src_workspace) + await FileRepository.save_file(filename=ctx.test_filename, content=TEST_CONTENT, relative_path=TEST_CODES_FILE_REPO) + output_data = RunCodeResult( + stdout=";", + stderr="", + summary="======================================================================\n" + "FAIL: test_player_calculate_score_with_multiple_aces (__main__.TestPlayer)\n" + "----------------------------------------------------------------------\n" + "Traceback (most recent call last):\n" + ' File "tests/test_player.py", line 46, in test_player_calculate_score_' + "with_multiple_aces\n" + " self.assertEqual(player.score, 12)\nAssertionError: 22 != 12\n\n" + "----------------------------------------------------------------------\n" + "Ran 5 tests in 0.007s\n\nFAILED (failures=1)\n;\n", + ) + await FileRepository.save_file( + filename=ctx.output_filename, content=output_data.json(), relative_path=TEST_OUTPUTS_FILE_REPO + ) + debug_error = DebugError(context=ctx) + + rsp = await debug_error.run() + + assert "class Player" in rsp # rewrite the same class + # a key logic to rewrite to (original one is "if self.score > 12") + assert "while self.score > 21" in rsp diff --git a/tests/metagpt/actions/test_design_api.py b/tests/metagpt/actions/test_design_api.py index 0add8fb74..e90707d1a 100644 --- a/tests/metagpt/actions/test_design_api.py +++ b/tests/metagpt/actions/test_design_api.py @@ -4,33 +4,27 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : test_design_api.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.design_api import WriteDesign +from metagpt.const import PRDS_FILE_REPO from metagpt.logs import logger from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository from tests.metagpt.actions.mock import PRD_SAMPLE @pytest.mark.asyncio async def test_design_api(): - prd = "我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。" + inputs = ["我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", PRD_SAMPLE] + for prd in inputs: + await FileRepository.save_file("new_prd.txt", content=prd, relative_path=PRDS_FILE_REPO) - design_api = WriteDesign("design_api") + design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) + result = await design_api.run([Message(content=prd, instruct_content=None)]) + logger.info(result) - assert result - - -@pytest.mark.asyncio -async def test_design_api_calculator(): - prd = PRD_SAMPLE - - design_api = WriteDesign("design_api") - result = await design_api.run([Message(content=prd, instruct_content=None)]) - logger.info(result) - - assert result + assert result diff --git a/tests/metagpt/actions/test_prepare_documents.py b/tests/metagpt/actions/test_prepare_documents.py new file mode 100644 index 000000000..31c8bcb80 --- /dev/null +++ b/tests/metagpt/actions/test_prepare_documents.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/6 +@Author : mashenquan +@File : test_prepare_documents.py +@Desc: Unit test for prepare_documents.py +""" +import pytest + +from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository + + +@pytest.mark.asyncio +async def test_prepare_documents(): + msg = Message(content="New user requirements balabala...") + + if CONFIG.git_repo: + CONFIG.git_repo.delete_repository() + CONFIG.git_repo = None + + await PrepareDocuments().run(with_messages=[msg]) + assert CONFIG.git_repo + doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) + assert doc + assert doc.content == msg.content diff --git a/tests/metagpt/actions/test_run_code.py b/tests/metagpt/actions/test_run_code.py index 1e451cb14..888418974 100644 --- a/tests/metagpt/actions/test_run_code.py +++ b/tests/metagpt/actions/test_run_code.py @@ -4,10 +4,12 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : test_run_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.run_code import RunCode +from metagpt.schema import RunCodeContext @pytest.mark.asyncio @@ -35,37 +37,29 @@ async def test_run_script(): @pytest.mark.asyncio async def test_run(): - action = RunCode() - result = await action.run(mode="text", code="print('Hello, World')") - assert "PASS" in result - - result = await action.run( - mode="script", - code="echo 'Hello World'", - code_file_name="", - test_code="", - test_file_name="", - command=["echo", "Hello World"], - working_directory=".", - additional_python_paths=[], - ) - assert "PASS" in result - - -@pytest.mark.asyncio -async def test_run_failure(): - action = RunCode() - result = await action.run(mode="text", code="result = 1 / 0") - assert "FAIL" in result - - result = await action.run( - mode="script", - code='python -c "print(1/0)"', - code_file_name="", - test_code="", - test_file_name="", - command=["python", "-c", "print(1/0)"], - working_directory=".", - additional_python_paths=[], - ) - assert "FAIL" in result + inputs = [ + (RunCodeContext(mode="text", code_filename="a.txt", code="print('Hello, World')"), "PASS"), + ( + RunCodeContext( + mode="script", + code_filename="a.sh", + code="echo 'Hello World'", + command=["echo", "Hello World"], + working_directory=".", + ), + "PASS", + ), + ( + RunCodeContext( + mode="script", + code_filename="a.py", + code='python -c "print(1/0)"', + command=["python", "-c", "print(1/0)"], + working_directory=".", + ), + "FAIL", + ), + ] + for ctx, result in inputs: + rsp = await RunCode(context=ctx).run() + assert result in rsp.summary diff --git a/tests/metagpt/actions/test_summarize_code.py b/tests/metagpt/actions/test_summarize_code.py new file mode 100644 index 000000000..7ecb67afd --- /dev/null +++ b/tests/metagpt/actions/test_summarize_code.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 17:46 +@Author : mashenquan +@File : test_summarize_code.py +@Modifiled By: mashenquan, 2023-12-6. Unit test for summarize_code.py +""" +import pytest + +from metagpt.actions.summarize_code import SummarizeCode +from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.logs import logger +from metagpt.schema import CodeSummarizeContext +from metagpt.utils.file_repository import FileRepository + +DESIGN_CONTENT = """ +{"Implementation approach": "To develop this snake game, we will use the Python language and choose the Pygame library. Pygame is an open-source Python module collection specifically designed for writing video games. It provides functionalities such as displaying images and playing sounds, making it suitable for creating intuitive and responsive user interfaces. We will ensure efficient game logic to prevent any delays during gameplay. The scoring system will be simple, with the snake gaining points for each food it eats. We will use Pygame's event handling system to implement pause and resume functionality, as well as high-score tracking. The difficulty will increase by speeding up the snake's movement. In the initial version, we will focus on single-player mode and consider adding multiplayer mode and customizable skins in future updates. Based on the new requirement, we will also add a moving obstacle that appears randomly. If the snake eats this obstacle, the game will end. If the snake does not eat the obstacle, it will disappear after 5 seconds. For this, we need to add mechanisms for obstacle generation, movement, and disappearance in the game logic.", "Project_name": "snake_game", "File list": ["main.py", "game.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "constants.py", "assets/styles.css", "assets/index.html"], "Data structures and interfaces": "```mermaid\n classDiagram\n class Game{\n +int score\n +int speed\n +bool game_over\n +bool paused\n +Snake snake\n +Food food\n +Obstacle obstacle\n +Scoreboard scoreboard\n +start_game() void\n +pause_game() void\n +resume_game() void\n +end_game() void\n +increase_difficulty() void\n +update() void\n +render() void\n Game()\n }\n class Snake{\n +list body_parts\n +str direction\n +bool grow\n +move() void\n +grow() void\n +check_collision() bool\n Snake()\n }\n class Food{\n +tuple position\n +spawn() void\n Food()\n }\n class Obstacle{\n +tuple position\n +int lifetime\n +bool active\n +spawn() void\n +move() void\n +check_collision() bool\n +disappear() void\n Obstacle()\n }\n class Scoreboard{\n +int high_score\n +update_score(int) void\n +reset_score() void\n +load_high_score() void\n +save_high_score() void\n Scoreboard()\n }\n class Constants{\n }\n Game \"1\" -- \"1\" Snake: has\n Game \"1\" -- \"1\" Food: has\n Game \"1\" -- \"1\" Obstacle: has\n Game \"1\" -- \"1\" Scoreboard: has\n ```", "Program call flow": "```sequenceDiagram\n participant M as Main\n participant G as Game\n participant S as Snake\n participant F as Food\n participant O as Obstacle\n participant SB as Scoreboard\n M->>G: start_game()\n loop game loop\n G->>S: move()\n G->>S: check_collision()\n G->>F: spawn()\n G->>O: spawn()\n G->>O: move()\n G->>O: check_collision()\n G->>O: disappear()\n G->>SB: update_score(score)\n G->>G: update()\n G->>G: render()\n alt if paused\n M->>G: pause_game()\n M->>G: resume_game()\n end\n alt if game_over\n G->>M: end_game()\n end\n end\n```", "Anything UNCLEAR": "There is no need for further clarification as the requirements are already clear."} +""" + +TASK_CONTENT = """ +{"Required Python third-party packages": ["pygame==2.0.1"], "Required Other language third-party packages": ["No third-party packages required for other languages."], "Full API spec": "\n openapi: 3.0.0\n info:\n title: Snake Game API\n version: \"1.0.0\"\n paths:\n /start:\n get:\n summary: Start the game\n responses:\n '200':\n description: Game started successfully\n /pause:\n get:\n summary: Pause the game\n responses:\n '200':\n description: Game paused successfully\n /resume:\n get:\n summary: Resume the game\n responses:\n '200':\n description: Game resumed successfully\n /end:\n get:\n summary: End the game\n responses:\n '200':\n description: Game ended successfully\n /score:\n get:\n summary: Get the current score\n responses:\n '200':\n description: Current score retrieved successfully\n /highscore:\n get:\n summary: Get the high score\n responses:\n '200':\n description: High score retrieved successfully\n components: {}\n ", "Logic Analysis": [["constants.py", "Contains all the constant values like screen size, colors, game speeds, etc. This should be implemented first as it provides the base values for other components."], ["snake.py", "Contains the Snake class with methods for movement, growth, and collision detection. It is dependent on constants.py for configuration values."], ["food.py", "Contains the Food class responsible for spawning food items on the screen. It is dependent on constants.py for configuration values."], ["obstacle.py", "Contains the Obstacle class with methods for spawning, moving, and disappearing of obstacles, as well as collision detection with the snake. It is dependent on constants.py for configuration values."], ["scoreboard.py", "Contains the Scoreboard class for updating, resetting, loading, and saving high scores. It may use constants.py for configuration values and depends on the game's scoring logic."], ["game.py", "Contains the main Game class which includes the game loop and methods for starting, pausing, resuming, and ending the game. It is dependent on snake.py, food.py, obstacle.py, and scoreboard.py."], ["main.py", "The entry point of the game that initializes the game and starts the game loop. It is dependent on game.py."]], "Task list": ["constants.py", "snake.py", "food.py", "obstacle.py", "scoreboard.py", "game.py", "main.py"], "Shared Knowledge": "\n 'constants.py' should contain all the necessary configurations for the game, such as screen dimensions, color definitions, and speed settings. These constants will be used across multiple files, ensuring consistency and ease of updates. Ensure that the Pygame library is initialized correctly in 'main.py' before starting the game loop. Also, make sure that the game's state is managed properly when pausing and resuming the game.\n ", "Anything UNCLEAR": "The interaction between the 'obstacle.py' and the game loop needs to be clearly defined to ensure obstacles appear and disappear correctly. The lifetime of the obstacle and its random movement should be implemented in a way that does not interfere with the game's performance."} +""" + +FOOD_PY = """ +## food.py +import random + +class Food: + def __init__(self): + self.position = (0, 0) + + def generate(self): + x = random.randint(0, 9) + y = random.randint(0, 9) + self.position = (x, y) + + def get_position(self): + return self.position + +""" + +GAME_PY = """ +## game.py +import pygame +from snake import Snake +from food import Food + +class Game: + def __init__(self): + self.score = 0 + self.level = 1 + self.snake = Snake() + self.food = Food() + + def start_game(self): + pygame.init() + self.initialize_game() + self.game_loop() + + def initialize_game(self): + self.score = 0 + self.level = 1 + self.snake.reset() + self.food.generate() + + def game_loop(self): + game_over = False + + while not game_over: + self.update() + self.draw() + self.handle_events() + self.check_collision() + self.increase_score() + self.increase_level() + + if self.snake.is_collision(): + game_over = True + self.game_over() + + def update(self): + self.snake.move() + + def draw(self): + self.snake.draw() + self.food.draw() + + def handle_events(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + pygame.quit() + quit() + elif event.type == pygame.KEYDOWN: + if event.key == pygame.K_UP: + self.snake.change_direction("UP") + elif event.key == pygame.K_DOWN: + self.snake.change_direction("DOWN") + elif event.key == pygame.K_LEFT: + self.snake.change_direction("LEFT") + elif event.key == pygame.K_RIGHT: + self.snake.change_direction("RIGHT") + + def check_collision(self): + if self.snake.get_head() == self.food.get_position(): + self.snake.grow() + self.food.generate() + + def increase_score(self): + self.score += 1 + + def increase_level(self): + if self.score % 10 == 0: + self.level += 1 + + def game_over(self): + print("Game Over") + self.initialize_game() + +""" + +MAIN_PY = """ +## main.py +import pygame +from game import Game + +def main(): + pygame.init() + game = Game() + game.start_game() + +if __name__ == "__main__": + main() + +""" + +SNAKE_PY = """ +## snake.py +import pygame + +class Snake: + def __init__(self): + self.body = [(0, 0)] + self.direction = (1, 0) + + def move(self): + head = self.body[0] + dx, dy = self.direction + new_head = (head[0] + dx, head[1] + dy) + self.body.insert(0, new_head) + self.body.pop() + + def change_direction(self, direction): + if direction == "UP": + self.direction = (0, -1) + elif direction == "DOWN": + self.direction = (0, 1) + elif direction == "LEFT": + self.direction = (-1, 0) + elif direction == "RIGHT": + self.direction = (1, 0) + + def grow(self): + tail = self.body[-1] + dx, dy = self.direction + new_tail = (tail[0] - dx, tail[1] - dy) + self.body.append(new_tail) + + def get_head(self): + return self.body[0] + + def get_body(self): + return self.body[1:] + +""" + + +@pytest.mark.asyncio +async def test_summarize_code(): + CONFIG.src_workspace = CONFIG.git_repo.workdir / "src" + await FileRepository.save_file(filename="1.json", relative_path=SYSTEM_DESIGN_FILE_REPO, content=DESIGN_CONTENT) + await FileRepository.save_file(filename="1.json", relative_path=TASK_FILE_REPO, content=TASK_CONTENT) + await FileRepository.save_file(filename="food.py", relative_path=CONFIG.src_workspace, content=FOOD_PY) + await FileRepository.save_file(filename="game.py", relative_path=CONFIG.src_workspace, content=GAME_PY) + await FileRepository.save_file(filename="main.py", relative_path=CONFIG.src_workspace, content=MAIN_PY) + await FileRepository.save_file(filename="snake.py", relative_path=CONFIG.src_workspace, content=SNAKE_PY) + + src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) + all_files = src_file_repo.all_files + ctx = CodeSummarizeContext(design_filename="1.json", task_filename="1.json", codes_filenames=all_files) + action = SummarizeCode(context=ctx) + rsp = await action.run() + assert rsp + logger.info(rsp) diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index eb5e3de91..54229089c 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,26 +4,31 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modifiled By: mashenquan, 2023-12-6. According to RFC 135 """ import pytest from metagpt.actions.write_code import WriteCode from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.schema import CodingContext, Document from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @pytest.mark.asyncio async def test_write_code(): - api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" - write_code = WriteCode("write_code") + context = CodingContext( + filename="task_filename.py", design_doc=Document(content="设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。") + ) + doc = Document(content=context.json()) + write_code = WriteCode(context=doc) - code = await write_code.run(api_design) - logger.info(code) + code = await write_code.run() + logger.info(code.json()) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 - assert "def add" in code - assert "return" in code + assert "def add" in code.code_doc.content + assert "return" in code.code_doc.content @pytest.mark.asyncio diff --git a/tests/metagpt/actions/test_write_code_review.py b/tests/metagpt/actions/test_write_code_review.py index 21bc563ec..e16eb7348 100644 --- a/tests/metagpt/actions/test_write_code_review.py +++ b/tests/metagpt/actions/test_write_code_review.py @@ -8,6 +8,8 @@ import pytest from metagpt.actions.write_code_review import WriteCodeReview +from metagpt.document import Document +from metagpt.schema import CodingContext @pytest.mark.asyncio @@ -16,13 +18,15 @@ async def test_write_code_review(capfd): def add(a, b): return a + """ - # write_code_review = WriteCodeReview("write_code_review") + context = CodingContext( + filename="math.py", design_doc=Document(content="编写一个从a加b的函数，返回a+b"), code_doc=Document(content=code) + ) - code = await WriteCodeReview().run(context="编写一个从a加b的函数，返回a+b", code=code, filename="math.py") + context = await WriteCodeReview(context=context).run() # 我们不能精确地预测生成的代码评审，但我们可以检查返回的是否为字符串 - assert isinstance(code, str) - assert len(code) > 0 + assert isinstance(context.code_doc.content, str) + assert len(context.code_doc.content) > 0 captured = capfd.readouterr() print(f"输出内容: {captured.out}") diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 8f8ef84f5..08be3cf75 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -9,19 +9,24 @@ import pytest from metagpt.actions import UserRequirement +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message +from metagpt.utils.file_repository import FileRepository @pytest.mark.asyncio async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" + await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=requirements, relative_path=DOCS_FILE_REPO) prd = await product_manager.run(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) # Assert the prd is not None or empty assert prd is not None - assert prd != "" + assert prd.content != "" + assert CONFIG.git_repo.new_file_repository(relative_path=PRDS_FILE_REPO).changed_files diff --git a/tests/metagpt/actions/test_write_test.py b/tests/metagpt/actions/test_write_test.py index e5acdff44..a3190fb0e 100644 --- a/tests/metagpt/actions/test_write_test.py +++ b/tests/metagpt/actions/test_write_test.py @@ -9,6 +9,7 @@ from metagpt.actions.write_test import WriteTest from metagpt.logs import logger +from metagpt.schema import Document, TestingContext @pytest.mark.asyncio @@ -24,22 +25,17 @@ def __init__(self, position: Tuple[int, int]): def generate(self, max_y: int, max_x: int): self.position = (random.randint(1, max_y - 1), random.randint(1, max_x - 1)) """ + context = TestingContext(filename="food.py", code_doc=Document(filename="food.py", content=code)) + write_test = WriteTest(context=context) - write_test = WriteTest() - - test_code = await write_test.run( - code_to_test=code, - test_file_name="test_food.py", - source_file_path="/some/dummy/path/cli_snake_game/cli_snake_game/food.py", - workspace="/some/dummy/path/cli_snake_game", - ) - logger.info(test_code) + context = await write_test.run() + logger.info(context.json()) # We cannot exactly predict the generated test cases, but we can check if it is a string and if it is not empty - assert isinstance(test_code, str) - assert "from cli_snake_game.food import Food" in test_code - assert "class TestFood(unittest.TestCase)" in test_code - assert "def test_generate" in test_code + assert isinstance(context.test_doc.content, str) + assert "from food import Food" in context.test_doc.content + assert "class TestFood(unittest.TestCase)" in context.test_doc.content + assert "def test_generate" in context.test_doc.content @pytest.mark.asyncio diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 5500b69f7..75f6b3b43 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -71,7 +71,7 @@ ``` ''' -SYSTEM_DESIGN = """## project_name +SYSTEM_DESIGN = """## Project name ```python "smart_search_engine" ``` diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py index a830b58aa..92e5204c5 100644 --- a/tests/metagpt/utils/test_file_repository.py +++ b/tests/metagpt/utils/test_file_repository.py @@ -43,6 +43,10 @@ async def test_file_repo(): assert {"a.txt"} == await file_repo.get_changed_dependency("b.txt") await file_repo.save("d/e.txt", "EEE") assert ["d/e.txt"] == file_repo.get_change_dir_files("d") + assert set(file_repo.all_files) == {"a.txt", "b.txt", "d/e.txt"} + await file_repo.delete("d/e.txt") + await file_repo.delete("d/e.txt") # delete twice + assert set(file_repo.all_files) == {"a.txt", "b.txt"} git_repo.delete_repository() From ec8c703c5a7b699880e73cba365fb41967489285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Dec 2023 19:55:47 +0800 Subject: [PATCH 204/232] feat: merge geekan:main --- examples/agent_creator.py | 1 + examples/search_kb.py | 22 ++++- metagpt/actions/action.py | 4 +- metagpt/actions/summarize_code.py | 4 +- metagpt/actions/write_code.py | 4 +- metagpt/actions/write_code_review.py | 4 +- metagpt/provider/openai_api.py | 6 +- metagpt/provider/zhipuai_api.py | 4 +- metagpt/roles/__init__.py | 2 +- metagpt/roles/sales.py | 2 +- metagpt/roles/{seacher.py => searcher.py} | 2 +- metagpt/subscription.py | 101 +++++++++++++++++++++ tests/conftest.py | 11 +++ tests/metagpt/test_subscription.py | 102 ++++++++++++++++++++++ 14 files changed, 251 insertions(+), 18 deletions(-) rename metagpt/roles/{seacher.py => searcher.py} (99%) create mode 100644 metagpt/subscription.py create mode 100644 tests/metagpt/test_subscription.py diff --git a/examples/agent_creator.py b/examples/agent_creator.py index e724105a3..05417d24a 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -49,6 +49,7 @@ def parse_code(rsp): pattern = r"```python(.*)```" match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" + CONFIG.workspace_path.mkdir(parents=True, exist_ok=True) with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text diff --git a/examples/search_kb.py b/examples/search_kb.py index 0b5d59385..7a9911ca2 100644 --- a/examples/search_kb.py +++ b/examples/search_kb.py @@ -5,17 +5,35 @@ """ import asyncio +from metagpt.actions import Action from metagpt.const import DATA_PATH from metagpt.document_store import FaissStore from metagpt.logs import logger from metagpt.roles import Sales +from metagpt.schema import Message + +""" example.json, e.g. +[ + { + "source": "Which facial cleanser is good for oily skin?", + "output": "ABC cleanser is preferred by many with oily skin." + }, + { + "source": "Is L'Oreal good to use?", + "output": "L'Oreal is a popular brand with many positive reviews." + } +] +""" async def search(): store = FaissStore(DATA_PATH / "example.json") role = Sales(profile="Sales", store=store) - - queries = ["Which facial cleanser is good for oily skin?", "Is L'Oreal good to use?"] + role._watch({Action}) + queries = [ + Message("Which facial cleanser is good for oily skin?", cause_by=Action), + Message("Is L'Oreal good to use?", cause_by=Action), + ] for query in queries: logger.info(f"User: {query}") result = await role.run(query) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index f8016b8a2..dc96699a9 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -9,7 +9,7 @@ from abc import ABC from typing import Optional -from tenacity import retry, stop_after_attempt, wait_fixed +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM @@ -53,7 +53,7 @@ async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> s system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) - @retry(stop=stop_after_attempt(3), wait=wait_fixed(1)) + @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) async def _aask_v1( self, prompt: str, diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index d10cd6c55..413ac2a21 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -7,7 +7,7 @@ """ from pathlib import Path -from tenacity import retry, stop_after_attempt, wait_fixed +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.config import CONFIG @@ -92,7 +92,7 @@ class SummarizeCode(Action): def __init__(self, name="SummarizeCode", context=None, llm=None): super().__init__(name, context, llm) - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60)) async def summarize_code(self, prompt): code_rsp = await self._aask(prompt) return code_rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 9b20843c7..4c138a124 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -15,7 +15,7 @@ RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ -from tenacity import retry, stop_after_attempt, wait_fixed +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.config import CONFIG @@ -81,7 +81,7 @@ class WriteCode(Action): def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=code_rsp) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index f7c6845d2..f9cebffac 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -8,7 +8,7 @@ WriteCode object, rather than passing them in when calling the run function. """ -from tenacity import retry, stop_after_attempt, wait_fixed +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.config import CONFIG @@ -94,7 +94,7 @@ class WriteCodeReview(Action): def __init__(self, name="WriteCodeReview", context=None, llm=None): super().__init__(name, context, llm) - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60)) async def write_code_review_and_rewrite(self, prompt): code_rsp = await self._aask(prompt) result = CodeParser.parse_block("Code Review Result", code_rsp) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 8ac0c4b21..a73bb0aa0 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -15,7 +15,7 @@ retry, retry_if_exception_type, stop_after_attempt, - wait_fixed, + wait_random_exponential, ) from metagpt.config import CONFIG @@ -231,8 +231,8 @@ async def acompletion(self, messages: list[dict]) -> dict: return await self._achat_completion(messages) @retry( - stop=stop_after_attempt(3), - wait=wait_fixed(1), + wait=wait_random_exponential(min=1, max=60), + stop=stop_after_attempt(6), after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index edd9084e3..92119b764 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -13,7 +13,7 @@ retry, retry_if_exception_type, stop_after_attempt, - wait_fixed, + wait_random_exponential, ) from metagpt.config import CONFIG @@ -122,7 +122,7 @@ async def _achat_completion_stream(self, messages: list[dict]) -> str: @retry( stop=stop_after_attempt(3), - wait=wait_fixed(1), + wait=wait_random_exponential(min=1, max=60), after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(ConnectionError), retry_error_callback=log_and_reraise, diff --git a/metagpt/roles/__init__.py b/metagpt/roles/__init__.py index 1768b786c..f033a5dfa 100644 --- a/metagpt/roles/__init__.py +++ b/metagpt/roles/__init__.py @@ -12,7 +12,7 @@ from metagpt.roles.product_manager import ProductManager from metagpt.roles.engineer import Engineer from metagpt.roles.qa_engineer import QaEngineer -from metagpt.roles.seacher import Searcher +from metagpt.roles.searcher import Searcher from metagpt.roles.sales import Sales from metagpt.roles.customer_service import CustomerService diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index 18282a494..d5aac1824 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -28,7 +28,7 @@ def __init__( def _set_store(self, store): if store: - action = SearchAndSummarize("", engine=SearchEngineType.CUSTOM_ENGINE, search_func=store.search) + action = SearchAndSummarize("", engine=SearchEngineType.CUSTOM_ENGINE, search_func=store.asearch) else: action = SearchAndSummarize() self._init_actions([action]) diff --git a/metagpt/roles/seacher.py b/metagpt/roles/searcher.py similarity index 99% rename from metagpt/roles/seacher.py rename to metagpt/roles/searcher.py index 587698d1d..bee8d3986 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/searcher.py @@ -3,7 +3,7 @@ """ @Time : 2023/5/23 17:25 @Author : alexanderwu -@File : seacher.py +@File : searcher.py @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ diff --git a/metagpt/subscription.py b/metagpt/subscription.py new file mode 100644 index 000000000..0d2b30821 --- /dev/null +++ b/metagpt/subscription.py @@ -0,0 +1,101 @@ +import asyncio +from typing import AsyncGenerator, Awaitable, Callable + +from pydantic import BaseModel, Field + +from metagpt.logs import logger +from metagpt.roles import Role +from metagpt.schema import Message + + +class SubscriptionRunner(BaseModel): + """A simple wrapper to manage subscription tasks for different roles using asyncio. + + Example: + >>> import asyncio + >>> from metagpt.subscription import SubscriptionRunner + >>> from metagpt.roles import Searcher + >>> from metagpt.schema import Message + + >>> async def trigger(): + ... while True: + ... yield Message("the latest news about OpenAI") + ... await asyncio.sleep(3600 * 24) + + >>> async def callback(msg: Message): + ... print(msg.content) + + >>> async def main(): + ... pb = SubscriptionRunner() + ... await pb.subscribe(Searcher(), trigger(), callback) + ... await pb.run() + + >>> asyncio.run(main()) + """ + + tasks: dict[Role, asyncio.Task] = Field(default_factory=dict) + + class Config: + arbitrary_types_allowed = True + + async def subscribe( + self, + role: Role, + trigger: AsyncGenerator[Message, None], + callback: Callable[ + [ + Message, + ], + Awaitable[None], + ], + ): + """Subscribes a role to a trigger and sets up a callback to be called with the role's response. + + Args: + role: The role to subscribe. + trigger: An asynchronous generator that yields Messages to be processed by the role. + callback: An asynchronous function to be called with the response from the role. + """ + loop = asyncio.get_running_loop() + + async def _start_role(): + async for msg in trigger: + resp = await role.run(msg) + await callback(resp) + + self.tasks[role] = loop.create_task(_start_role(), name=f"Subscription-{role}") + + async def unsubscribe(self, role: Role): + """Unsubscribes a role from its trigger and cancels the associated task. + + Args: + role: The role to unsubscribe. + """ + task = self.tasks.pop(role) + task.cancel() + + async def run(self, raise_exception: bool = True): + """Runs all subscribed tasks and handles their completion or exception. + + Args: + raise_exception: _description_. Defaults to True. + + Raises: + task.exception: _description_ + """ + while True: + for role, task in self.tasks.items(): + if task.done(): + if task.exception(): + if raise_exception: + raise task.exception() + logger.opt(exception=task.exception()).error(f"Task {task.get_name()} run error") + else: + logger.warning( + f"Task {task.get_name()} has completed. " + "If this is unexpected behavior, please check the trigger function." + ) + self.tasks.pop(role) + break + else: + await asyncio.sleep(1) diff --git a/tests/conftest.py b/tests/conftest.py index 8e4422700..0cef6a4c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -73,6 +73,17 @@ async def handle_client(reader, writer): return "http://{}:{}".format(*server.sockets[0].getsockname()) +# see https://github.com/Delgan/loguru/issues/59#issuecomment-466591978 +@pytest.fixture +def loguru_caplog(caplog): + class PropogateHandler(logging.Handler): + def emit(self, record): + logging.getLogger(record.name).handle(record) + + logger.add(PropogateHandler(), format="{message}") + yield caplog + + # init & dispose git repo @pytest.fixture(scope="session", autouse=True) def setup_and_teardown_git_repo(request): diff --git a/tests/metagpt/test_subscription.py b/tests/metagpt/test_subscription.py new file mode 100644 index 000000000..2e898424d --- /dev/null +++ b/tests/metagpt/test_subscription.py @@ -0,0 +1,102 @@ +import asyncio + +import pytest + +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.subscription import SubscriptionRunner + + +@pytest.mark.asyncio +async def test_subscription_run(): + callback_done = 0 + + async def trigger(): + while True: + yield Message("the latest news about OpenAI") + await asyncio.sleep(3600 * 24) + + class MockRole(Role): + async def run(self, message=None): + return Message("") + + async def callback(message): + nonlocal callback_done + callback_done += 1 + + runner = SubscriptionRunner() + + roles = [] + for _ in range(2): + role = MockRole() + roles.append(role) + await runner.subscribe(role, trigger(), callback) + + task = asyncio.get_running_loop().create_task(runner.run()) + + for _ in range(10): + if callback_done == 2: + break + await asyncio.sleep(0) + else: + raise TimeoutError("callback not call") + + role = roles[0] + assert role in runner.tasks + await runner.unsubscribe(roles[0]) + + for _ in range(10): + if role not in runner.tasks: + break + await asyncio.sleep(0) + else: + raise TimeoutError("callback not call") + + task.cancel() + for i in runner.tasks.values(): + i.cancel() + + +@pytest.mark.asyncio +async def test_subscription_run_error(loguru_caplog): + async def trigger1(): + while True: + yield Message("the latest news about OpenAI") + await asyncio.sleep(3600 * 24) + + async def trigger2(): + yield Message("the latest news about OpenAI") + + class MockRole1(Role): + async def run(self, message=None): + raise RuntimeError + + class MockRole2(Role): + async def run(self, message=None): + return Message("") + + async def callback(msg: Message): + print(msg) + + runner = SubscriptionRunner() + await runner.subscribe(MockRole1(), trigger1(), callback) + with pytest.raises(RuntimeError): + await runner.run() + + await runner.subscribe(MockRole2(), trigger2(), callback) + task = asyncio.get_running_loop().create_task(runner.run(False)) + + for _ in range(10): + if not runner.tasks: + break + await asyncio.sleep(0) + else: + raise TimeoutError("wait runner tasks empty timeout") + + task.cancel() + for i in runner.tasks.values(): + i.cancel() + assert len(loguru_caplog.records) >= 2 + logs = "".join(loguru_caplog.messages) + assert "run error" in logs + assert "has completed" in logs From b4eb8e4f34ed3efa11c4fd817cc49a7679a2b50b Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 11 Dec 2023 14:58:54 +0800 Subject: [PATCH 205/232] use metagpt cli instead. update all related docs --- README.md | 7 ++++--- docs/FAQ-EN.md | 2 +- docs/README_CN.md | 6 +++--- docs/README_JA.md | 24 ++++++++++++------------ docs/install/docker_install.md | 6 +++--- docs/install/docker_install_cn.md | 6 +++--- docs/tutorial/usage.md | 12 ++++++------ docs/tutorial/usage_cn.md | 10 +++++----- metagpt/actions/write_docstring.py | 2 +- tests/metagpt/test_startup.py | 1 + 10 files changed, 39 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index e80082a3a..2ce768212 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # MetaGPT: The Multi-Agent Framework

@@ -50,9 +51,9 @@ git clone https://github.com/geekan/MetaGPT.git cd MetaGPT pip3 install -e. # or pip3 install metagpt # for stable version -# Step 3: run the startup.py +# Step 3: run metagpt cli # setup your OPENAI_API_KEY in key.yaml copy from config.yaml -python3 startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # Step 4 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step. # If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) @@ -78,7 +79,7 @@ docker run --rm \ -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" ``` detail installation please refer to [docker_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-with-docker) diff --git a/docs/FAQ-EN.md b/docs/FAQ-EN.md index f9df50caf..b87e5da1e 100644 --- a/docs/FAQ-EN.md +++ b/docs/FAQ-EN.md @@ -98,7 +98,7 @@ MetaGPT Community - The position of Chief Evangelist rotates on a monthly basis. 1. How to change the investment amount? - 1. You can view all commands by typing `python startup.py --help` + 1. You can view all commands by typing `metagpt --help` 1. Which version of Python is more stable? diff --git a/docs/README_CN.md b/docs/README_CN.md index 038925184..1e0edc533 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -47,9 +47,9 @@ git clone https://github.com/geekan/MetaGPT.git cd MetaGPT pip3 install -e. # 或者 pip3 install metagpt # 安装稳定版本 -# 第 3 步：执行startup.py +# 第 3 步：执行metagpt # 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY -python3 startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # 第 4 步【可选的】：如果你想在执行过程中保存像象限图、系统设计、序列流程等图表这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 # 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js @@ -75,7 +75,7 @@ docker run --rm \ -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" ``` 详细的安装请安装 [docker_install](https://docs.deepwisdom.ai/zhcn/guide/get_started/installation.html#%E4%BD%BF%E7%94%A8docker%E5%AE%89%E8%A3%85) diff --git a/docs/README_JA.md b/docs/README_JA.md index 411d190b4..210044ec2 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -41,7 +41,7 @@ https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace ## 例（GPT-4 で完全生成） -例えば、`python startup.py "Toutiao のような RecSys をデザインする"`と入力すると、多くの出力が得られます +例えば、`metagpt "Toutiao のような RecSys をデザインする"`と入力すると、多くの出力が得られます ![Jinri Toutiao Recsys データと API デザイン](resources/workspace/content_rec_sys/resources/data_api_design.png) @@ -67,9 +67,9 @@ git clone https://github.com/geekan/MetaGPT.git cd MetaGPT pip install -e. -# ステップ 3: startup.py を実行する +# ステップ 3: metagpt を実行する # config.yaml を key.yaml にコピーし、独自の OPENAI_API_KEY を設定します -python3 startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # ステップ 4 [オプション]: 実行中に PRD ファイルなどのアーティファクトを保存する場合は、ステップ 3 の前にこのステップを実行できます。デフォルトでは、フレームワークには互換性があり、この手順を実行しなくてもプロセス全体を完了できます。 # NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） @@ -178,7 +178,7 @@ docker run --rm \ -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" # コンテナを起動し、その中でコマンドを実行することもできます docker run --name metagpt -d \ @@ -188,7 +188,7 @@ docker run --name metagpt -d \ metagpt/metagpt:latest docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" +$ metagpt "Write a cli snake game" ``` コマンド `docker run ...` は以下のことを行います: @@ -196,7 +196,7 @@ $ python startup.py "Write a cli snake game" - 特権モードで実行し、ブラウザの実行権限を得る - ホスト設定ファイル `/opt/metagpt/config/key.yaml` をコンテナ `/app/metagpt/config/key.yaml` にマップします - ホストディレクトリ `/opt/metagpt/workspace` をコンテナディレクトリ `/app/metagpt/workspace` にマップするs -- デモコマンド `python startup.py "Write a cli snake game"` を実行する +- デモコマンド `metagpt "Write a cli snake game"` を実行する ### 自分でイメージをビルドする @@ -225,11 +225,11 @@ cp config/config.yaml config/key.yaml ```shell # スクリプトの実行 -python startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # プロジェクトの実施にエンジニアを雇わないこと -python startup.py "Write a cli snake game" --implement False +metagpt "Write a cli snake game" --implement False # エンジニアを雇い、コードレビューを行う -python startup.py "Write a cli snake game" --code_review True +metagpt "Write a cli snake game" --code_review True ``` スクリプトを実行すると、`workspace/` ディレクトリに新しいプロジェクトが見つかります。 @@ -239,17 +239,17 @@ python startup.py "Write a cli snake game" --code_review True 要件を述べるときに、どのプラットフォームまたはツールを使用するかを指定できます。 ```shell -python startup.py "pygame をベースとした cli ヘビゲームを書く" +metagpt "pygame をベースとした cli ヘビゲームを書く" ``` ### 使用方法 ``` 会社名 - startup.py - 私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。 + metagpt - 私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。 シノプシス - startup.py IDEA + metagpt IDEA 説明 私たちは AI で構成されたソフトウェア・スタートアップです。私たちに投資することは、無限の可能性に満ちた未来に力を与えることです。 diff --git a/docs/install/docker_install.md b/docs/install/docker_install.md index b803a5dae..37125bdbe 100644 --- a/docs/install/docker_install.md +++ b/docs/install/docker_install.md @@ -15,7 +15,7 @@ docker run --rm \ -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python3 startup.py "Write a cli snake game" + metagpt "Write a cli snake game" # You can also start a container and execute commands in it docker run --name metagpt -d \ @@ -25,7 +25,7 @@ docker run --name metagpt -d \ metagpt/metagpt:latest docker exec -it metagpt /bin/bash -$ python3 startup.py "Write a cli snake game" +$ metagpt "Write a cli snake game" ``` The command `docker run ...` do the following things: @@ -33,7 +33,7 @@ The command `docker run ...` do the following things: - Run in privileged mode to have permission to run the browser - Map host configure file `/opt/metagpt/config/key.yaml` to container `/app/metagpt/config/key.yaml` - Map host directory `/opt/metagpt/workspace` to container `/app/metagpt/workspace` -- Execute the demo command `python3 startup.py "Write a cli snake game"` +- Execute the demo command `metagpt "Write a cli snake game"` ### Build image by yourself diff --git a/docs/install/docker_install_cn.md b/docs/install/docker_install_cn.md index 347fae10c..f360b49ed 100644 --- a/docs/install/docker_install_cn.md +++ b/docs/install/docker_install_cn.md @@ -15,7 +15,7 @@ docker run --rm \ -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ - python startup.py "Write a cli snake game" + metagpt "Write a cli snake game" # 您也可以启动一个容器并在其中执行命令 docker run --name metagpt -d \ @@ -25,7 +25,7 @@ docker run --name metagpt -d \ metagpt/metagpt:latest docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" +$ metagpt "Write a cli snake game" ``` `docker run ...`做了以下事情: @@ -33,7 +33,7 @@ $ python startup.py "Write a cli snake game" - 以特权模式运行，有权限运行浏览器 - 将主机文件 `/opt/metagpt/config/key.yaml` 映射到容器文件 `/app/metagpt/config/key.yaml` - 将主机目录 `/opt/metagpt/workspace` 映射到容器目录 `/app/metagpt/workspace` -- 执行示例命令 `python startup.py "Write a cli snake game"` +- 执行示例命令 `metagpt "Write a cli snake game"` ### 自己构建镜像 diff --git a/docs/tutorial/usage.md b/docs/tutorial/usage.md index ee87b65c9..f3eb931f6 100644 --- a/docs/tutorial/usage.md +++ b/docs/tutorial/usage.md @@ -19,11 +19,11 @@ cp config/config.yaml config/key.yaml ```shell # Run the script -python startup.py "Write a cli snake game" +metagpt "Write a cli snake game" # Do not hire an engineer to implement the project -python startup.py "Write a cli snake game" --implement False +metagpt "Write a cli snake game" --implement False # Hire an engineer and perform code reviews -python startup.py "Write a cli snake game" --code_review True +metagpt "Write a cli snake game" --code_review True ``` After running the script, you can find your new project in the `workspace/` directory. @@ -33,17 +33,17 @@ After running the script, you can find your new project in the `workspace/` dire You can tell which platform or tool you want to use when stating your requirements. ```shell -python startup.py "Write a cli snake game based on pygame" +metagpt "Write a cli snake game based on pygame" ``` ### Usage ``` NAME - startup.py - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + metagpt - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. SYNOPSIS - startup.py IDEA + metagpt IDEA DESCRIPTION We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. diff --git a/docs/tutorial/usage_cn.md b/docs/tutorial/usage_cn.md index 4b3bdd2c3..18966acdc 100644 --- a/docs/tutorial/usage_cn.md +++ b/docs/tutorial/usage_cn.md @@ -18,9 +18,9 @@ cp config/config.yaml config/key.yaml ### 示例：启动一个创业公司 ```shell -python startup.py "写一个命令行贪吃蛇" +metagpt "写一个命令行贪吃蛇" # 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 -python startup.py "写一个命令行贪吃蛇" --code_review True +metagpt "写一个命令行贪吃蛇" --code_review True ``` 运行脚本后，您可以在 `workspace/` 目录中找到您的新项目。 @@ -29,17 +29,17 @@ python startup.py "写一个命令行贪吃蛇" --code_review True 可以在阐述需求时说明想要使用的平台或工具。 例如： ```shell -python startup.py "写一个基于pygame的命令行贪吃蛇" +metagpt "写一个基于pygame的命令行贪吃蛇" ``` ### 使用 ``` 名称 - startup.py - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 + metagpt - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 概要 - startup.py IDEA + metagpt IDEA 描述 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 diff --git a/metagpt/actions/write_docstring.py b/metagpt/actions/write_docstring.py index dd3312bd5..0ad134157 100644 --- a/metagpt/actions/write_docstring.py +++ b/metagpt/actions/write_docstring.py @@ -16,7 +16,7 @@ Default: 'google' Example: - python3 -m metagpt.actions.write_docstring startup.py --overwrite False --style=numpy + python3 -m metagpt.actions.write_docstring ./metagpt/startup.py --overwrite False --style=numpy This script uses the 'fire' library to create a command-line interface. It generates docstrings for the given Python code using the specified docstring style and adds them to the code. diff --git a/tests/metagpt/test_startup.py b/tests/metagpt/test_startup.py index 53d3509ed..c34fd2c31 100644 --- a/tests/metagpt/test_startup.py +++ b/tests/metagpt/test_startup.py @@ -16,6 +16,7 @@ @pytest.mark.asyncio async def test_team(): + # FIXME: we're now using "metagpt" cli, so the entrance should be replaced instead. company = Team() company.run_project("做一个基础搜索引擎，可以支持知识库") history = await company.run(n_round=5) From 9a361593ea1d94081e244e73f8c11ebc24b3931a Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 11 Dec 2023 15:17:27 +0800 Subject: [PATCH 206/232] use metagpt cli instead. update all related docs --- docs/README_JA.md | 4 ++-- docs/tutorial/usage.md | 4 ++-- docs/tutorial/usage_cn.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/README_JA.md b/docs/README_JA.md index 210044ec2..63894647e 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -227,9 +227,9 @@ cp config/config.yaml config/key.yaml # スクリプトの実行 metagpt "Write a cli snake game" # プロジェクトの実施にエンジニアを雇わないこと -metagpt "Write a cli snake game" --implement False +metagpt "Write a cli snake game" --no-implement # エンジニアを雇い、コードレビューを行う -metagpt "Write a cli snake game" --code_review True +metagpt "Write a cli snake game" --code_review ``` スクリプトを実行すると、`workspace/` ディレクトリに新しいプロジェクトが見つかります。 diff --git a/docs/tutorial/usage.md b/docs/tutorial/usage.md index f3eb931f6..fbe4a8311 100644 --- a/docs/tutorial/usage.md +++ b/docs/tutorial/usage.md @@ -21,9 +21,9 @@ cp config/config.yaml config/key.yaml # Run the script metagpt "Write a cli snake game" # Do not hire an engineer to implement the project -metagpt "Write a cli snake game" --implement False +metagpt "Write a cli snake game" --no-implement # Hire an engineer and perform code reviews -metagpt "Write a cli snake game" --code_review True +metagpt "Write a cli snake game" --code_review ``` After running the script, you can find your new project in the `workspace/` directory. diff --git a/docs/tutorial/usage_cn.md b/docs/tutorial/usage_cn.md index 18966acdc..1ef50d633 100644 --- a/docs/tutorial/usage_cn.md +++ b/docs/tutorial/usage_cn.md @@ -20,7 +20,7 @@ cp config/config.yaml config/key.yaml ```shell metagpt "写一个命令行贪吃蛇" # 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 -metagpt "写一个命令行贪吃蛇" --code_review True +metagpt "写一个命令行贪吃蛇" --code_review ``` 运行脚本后，您可以在 `workspace/` 目录中找到您的新项目。 From 292344cf40959bb6ddadfe2ae7862c48811dd838 Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 11 Dec 2023 15:23:55 +0800 Subject: [PATCH 207/232] change all mail address from fuzhi.ai to deepwisdom.ai --- README.md | 2 +- docs/README_CN.md | 2 +- docs/README_JA.md | 2 +- metagpt/tools/sd_engine.py | 2 +- setup.py | 2 +- tests/metagpt/actions/test_ui_design.py | 2 +- tests/metagpt/roles/test_ui.py | 2 +- tests/metagpt/roles/ui_role.py | 2 +- tests/metagpt/tools/test_sd_tool.py | 2 +- tests/metagpt/tools/test_web_browser_engine.py | 4 ++-- tests/metagpt/tools/test_web_browser_engine_playwright.py | 6 +++--- tests/metagpt/tools/test_web_browser_engine_selenium.py | 6 +++--- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 2ce768212..b3473a12c 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Looking forward to seeing you there! 🎉 If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions! -- **Email:** alexanderwu@fuzhi.ai +- **Email:** alexanderwu@deepwisdom.ai - **GitHub Issues:** For more technical inquiries, you can also create a new issue in our [GitHub repository](https://github.com/geekan/metagpt/issues). We will respond to all questions within 2-3 business days. diff --git a/docs/README_CN.md b/docs/README_CN.md index 1e0edc533..dd65c2a25 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -114,7 +114,7 @@ https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace 如果您对这个项目有任何问题或反馈，欢迎联系我们。我们非常欢迎您的建议！ -- **邮箱：** alexanderwu@fuzhi.ai +- **邮箱：** alexanderwu@deepwisdom.ai - **GitHub 问题：** 对于更技术性的问题，您也可以在我们的 [GitHub 仓库](https://github.com/geekan/metagpt/issues) 中创建一个新的问题。 我们会在2-3个工作日内回复所有问题。 diff --git a/docs/README_JA.md b/docs/README_JA.md index 63894647e..482b42fa7 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -317,7 +317,7 @@ Hugging Face Space で試す このプロジェクトに関するご質問やご意見がございましたら、お気軽にお問い合わせください。皆様のご意見をお待ちしております！ -- **Email:** alexanderwu@fuzhi.ai +- **Email:** alexanderwu@deepwisdom.ai - **GitHub Issues:** 技術的なお問い合わせについては、[GitHub リポジトリ](https://github.com/geekan/metagpt/issues) に新しい issue を作成することもできます。 ご質問には 2-3 営業日以内に回答いたします。 diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index c6676a247..a84812f7c 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/19 16:28 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # @Desc : import asyncio import base64 diff --git a/setup.py b/setup.py index 6d3708c32..84e91ede8 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def run(self): long_description_content_type="text/markdown", url="https://github.com/geekan/MetaGPT", author="Alexander Wu", - author_email="alexanderwu@fuzhi.ai", + author_email="alexanderwu@deepwisdom.ai", license="MIT", keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming", packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), diff --git a/tests/metagpt/actions/test_ui_design.py b/tests/metagpt/actions/test_ui_design.py index b8be914ae..83590ec7d 100644 --- a/tests/metagpt/actions/test_ui_design.py +++ b/tests/metagpt/actions/test_ui_design.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/22 02:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # from tests.metagpt.roles.ui_role import UIDesign diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index 5904bee8f..2038a1aee 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/22 02:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # from metagpt.roles import ProductManager from metagpt.team import Team diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index ee36befbd..8ac799bf3 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/15 16:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # @Desc : import os import re diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index edb23df42..e457101a9 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/22 02:40 -# @Author : stellahong (stellahong@fuzhi.ai) +# @Author : stellahong (stellahong@deepwisdom.ai) # import os diff --git a/tests/metagpt/tools/test_web_browser_engine.py b/tests/metagpt/tools/test_web_browser_engine.py index b08d0ca10..28dd0e15c 100644 --- a/tests/metagpt/tools/test_web_browser_engine.py +++ b/tests/metagpt/tools/test_web_browser_engine.py @@ -7,8 +7,8 @@ @pytest.mark.parametrize( "browser_type, url, urls", [ - (WebBrowserEngineType.PLAYWRIGHT, "https://fuzhi.ai", ("https://fuzhi.ai",)), - (WebBrowserEngineType.SELENIUM, "https://fuzhi.ai", ("https://fuzhi.ai",)), + (WebBrowserEngineType.PLAYWRIGHT, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + (WebBrowserEngineType.SELENIUM, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), ], ids=["playwright", "selenium"], ) diff --git a/tests/metagpt/tools/test_web_browser_engine_playwright.py b/tests/metagpt/tools/test_web_browser_engine_playwright.py index 69e1339e7..e9ea80b10 100644 --- a/tests/metagpt/tools/test_web_browser_engine_playwright.py +++ b/tests/metagpt/tools/test_web_browser_engine_playwright.py @@ -8,9 +8,9 @@ @pytest.mark.parametrize( "browser_type, use_proxy, kwagrs, url, urls", [ - ("chromium", {"proxy": True}, {}, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("firefox", {}, {"ignore_https_errors": True}, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("webkit", {}, {"ignore_https_errors": True}, "https://fuzhi.ai", ("https://fuzhi.ai",)), + ("chromium", {"proxy": True}, {}, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("firefox", {}, {"ignore_https_errors": True}, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("webkit", {}, {"ignore_https_errors": True}, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), ], ids=["chromium-normal", "firefox-normal", "webkit-normal"], ) diff --git a/tests/metagpt/tools/test_web_browser_engine_selenium.py b/tests/metagpt/tools/test_web_browser_engine_selenium.py index ce322f7bd..ac6eafee7 100644 --- a/tests/metagpt/tools/test_web_browser_engine_selenium.py +++ b/tests/metagpt/tools/test_web_browser_engine_selenium.py @@ -8,9 +8,9 @@ @pytest.mark.parametrize( "browser_type, use_proxy, url, urls", [ - ("chrome", True, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("firefox", False, "https://fuzhi.ai", ("https://fuzhi.ai",)), - ("edge", False, "https://fuzhi.ai", ("https://fuzhi.ai",)), + ("chrome", True, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("firefox", False, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), + ("edge", False, "https://deepwisdom.ai", ("https://deepwisdom.ai",)), ], ids=["chrome-normal", "firefox-normal", "edge-normal"], ) From 687e17367c9bcad10b54fd9af8afbcc74ef42433 Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 11 Dec 2023 16:07:53 +0800 Subject: [PATCH 208/232] use python3 instead of python --- docs/FAQ-EN.md | 2 +- docs/README_JA.md | 2 +- docs/install/cli_install_cn.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/FAQ-EN.md b/docs/FAQ-EN.md index b87e5da1e..af6868509 100644 --- a/docs/FAQ-EN.md +++ b/docs/FAQ-EN.md @@ -134,7 +134,7 @@ MetaGPT Community - The position of Chief Evangelist rotates on a monthly basis. 1. Configuration instructions for SD Skills: The SD interface is currently deployed based on *https://github.com/AUTOMATIC1111/stable-diffusion-webui* **For environmental configurations and model downloads, please refer to the aforementioned GitHub repository. To initiate the SD service that supports API calls, run the command specified in cmd with the parameter nowebui, i.e., - 1. > python webui.py --enable-insecure-extension-access --port xxx --no-gradio-queue --nowebui + 1. > python3 webui.py --enable-insecure-extension-access --port xxx --no-gradio-queue --nowebui 1.     Once it runs without errors, the interface will be accessible after approximately 1 minute when the model finishes loading. 1. Configure SD_URL and SD_T2I_API in the config.yaml/key.yaml files. 1. ![](https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/065295a67b0b4feea665d1372722d49d~tplv-k3u1fbpfcp-zoom-1.image) diff --git a/docs/README_JA.md b/docs/README_JA.md index 482b42fa7..05f718635 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -60,7 +60,7 @@ https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace ```bash # ステップ 1: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: -python --version +python3 --version # ステップ 2: リポジトリをローカルマシンにクローンし、インストールする。 git clone https://github.com/geekan/MetaGPT.git diff --git a/docs/install/cli_install_cn.md b/docs/install/cli_install_cn.md index f351090ed..b1da1b813 100644 --- a/docs/install/cli_install_cn.md +++ b/docs/install/cli_install_cn.md @@ -15,7 +15,7 @@ npm --version sudo npm install -g @mermaid-js/mermaid-cli # 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： -python --version +python3 --version # 第 3 步：克隆仓库到您的本地机器，并进行安装。 git clone https://github.com/geekan/MetaGPT.git From 697f790c837f248321d0c7705da6c5ba7d840897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Dec 2023 16:42:08 +0800 Subject: [PATCH 209/232] bugfix: write code add related code file context --- metagpt/actions/write_code.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 4c138a124..b20539e78 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -14,12 +14,13 @@ 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ +import json from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO +from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser @@ -101,10 +102,11 @@ async def run(self, *args, **kwargs) -> CodingContext: if test_doc: test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr + code_context = await self._get_codes(coding_context.task_doc) prompt = PROMPT_TEMPLATE.format( design=coding_context.design_doc.content, tasks=coding_context.task_doc.content if coding_context.task_doc else "", - code=coding_context.code_doc.content if coding_context.code_doc else "", + code=code_context, logs=logs, filename=self.context.filename, summary_log=summary_doc.content if summary_doc else "", @@ -115,3 +117,21 @@ async def run(self, *args, **kwargs) -> CodingContext: coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace) coding_context.code_doc.content = code return coding_context + + @staticmethod + async def _get_codes(task_doc) -> str: + if not task_doc: + return "" + if not task_doc.content: + task_doc.content = FileRepository.get_file(filename=task_doc.filename, relative_path=TASK_FILE_REPO) + m = json.loads(task_doc.content) + code_filenames = m.get("Task list", []) + codes = [] + src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) + for filename in code_filenames: + doc = await src_file_repo.get(filename=filename) + if not doc: + continue + codes.append(doc.content) + return "\n----------\n".join(codes) + From 4cb3485c86bbfe3f96fb00b6bb0c15a6244a2282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 12 Dec 2023 21:32:03 +0800 Subject: [PATCH 210/232] feat: After users provide bug feedback, move directly to the WriteCode stage of the process. --- metagpt/actions/fix_bug.py | 14 +++++++ metagpt/actions/write_code.py | 11 ++++- metagpt/actions/write_prd.py | 49 +++++++++++++++++++--- metagpt/const.py | 1 + metagpt/roles/engineer.py | 11 ++--- metagpt/roles/role.py | 2 + metagpt/schema.py | 20 +++++---- metagpt/utils/git_repository.py | 42 ++++++++++++++++--- requirements.txt | 2 +- tests/metagpt/utils/test_git_repository.py | 7 ++++ 10 files changed, 132 insertions(+), 27 deletions(-) create mode 100644 metagpt/actions/fix_bug.py diff --git a/metagpt/actions/fix_bug.py b/metagpt/actions/fix_bug.py new file mode 100644 index 000000000..6bd550d3d --- /dev/null +++ b/metagpt/actions/fix_bug.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023-12-12 +@Author : mashenquan +@File : fix_bug.py +""" +from metagpt.actions import Action + + +class FixBug(Action): + """Fix bug action without any implementation details""" + + async def run(self, *args, **kwargs): + raise NotImplementedError diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index b20539e78..1dda6466f 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -20,7 +20,8 @@ from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO +from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO, BUGFIX_FILENAME, \ + DOCS_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser @@ -55,6 +56,12 @@ {summary_log} ``` ----- +# Bug Feedback logs +```text +{feedback} +``` +----- + ## Code: {filename} Write code with triple quoto, based on the following list and context. 1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. @@ -89,6 +96,7 @@ async def write_code(self, prompt) -> str: return code async def run(self, *args, **kwargs) -> CodingContext: + bug_feedback = await FileRepository.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO) coding_context = CodingContext.loads(self.context.content) test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO @@ -108,6 +116,7 @@ async def run(self, *args, **kwargs) -> CodingContext: tasks=coding_context.task_doc.content if coding_context.task_doc else "", code=code_context, logs=logs, + feedback=bug_feedback.content if bug_feedback else "", filename=self.context.filename, summary_log=summary_doc.content if summary_doc else "", ) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 530a22def..aad2422ef 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -17,6 +17,7 @@ from typing import List from metagpt.actions import Action, ActionOutput +from metagpt.actions.fix_bug import FixBug from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG from metagpt.const import ( @@ -24,10 +25,10 @@ DOCS_FILE_REPO, PRD_PDF_FILE_REPO, PRDS_FILE_REPO, - REQUIREMENT_FILENAME, + REQUIREMENT_FILENAME, BUGFIX_FILENAME, ) from metagpt.logs import logger -from metagpt.schema import Document, Documents +from metagpt.schema import Document, Documents, Message, BugFixContext from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template @@ -227,7 +228,6 @@ }, } - OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), @@ -305,15 +305,44 @@ and only output the json inside this tag, nothing else """ +IS_BUGFIX_PROMPT = """ +{content} + +___ +You are a professional product manager; You need to determine whether the above content describes a requirement or provides feedback about a bug. +Respond with `YES` if it is a feedback about a bug, `NO` if it is not, and provide the reasons. Return the response in JSON format like below: + +```json +{{ + "is_bugfix": ..., # `YES` or `NO` + "reason": ..., # reason string +}} +``` +""" + class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput | Message: # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are # related to the PRD. If they are related, rewrite the PRD. - requirement_doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) + docs_file_repo = CONFIG.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO) + requirement_doc = await docs_file_repo.get(filename=REQUIREMENT_FILENAME) + if await self._is_bugfix(requirement_doc.content): + await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) + await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") + bug_fix = BugFixContext(filename=BUGFIX_FILENAME) + return Message(content=bug_fix.json(), instruct_content=bug_fix, + role=self.profile, + cause_by=FixBug, + sent_from=self, + send_to="Alex", # the name of Engineer + ) + else: + await docs_file_repo.delete(filename=BUGFIX_FILENAME) + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) prd_docs = await prds_file_repo.get_all() change_files = Documents() @@ -405,7 +434,7 @@ async def _save_competitive_analysis(prd_doc): if not quadrant_chart: return pathname = ( - CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") ) if not pathname.parent.exists(): pathname.parent.mkdir(parents=True, exist_ok=True) @@ -430,3 +459,11 @@ async def _rename_workspace(prd): ws_name = CodeParser.parse_str(block="Project Name", text=prd) CONFIG.project_name = ws_name CONFIG.git_repo.rename_root(CONFIG.project_name) + + async def _is_bugfix(self, content): + prompt = IS_BUGFIX_PROMPT.format(content=content) + res = await self._aask(prompt=prompt) + logger.info(f"IS_BUGFIX:{res}") + if "YES" in res: + return True + return False diff --git a/metagpt/const.py b/metagpt/const.py index bd735a5e1..f6f64a27d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -74,6 +74,7 @@ def get_metagpt_root(): MESSAGE_ROUTE_TO_NONE = "" REQUIREMENT_FILENAME = "requirement.txt" +BUGFIX_FILENAME = "bugfix.txt" PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt" DOCS_FILE_REPO = "docs" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 9f8eb6482..cedd2101f 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -24,6 +24,7 @@ from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks +from metagpt.actions.fix_bug import FixBug from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( @@ -78,7 +79,7 @@ def __init__( """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -191,14 +192,14 @@ async def _is_pass(self, summary) -> (str, str): async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode, FixBug]) summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if not self._rc.news: return None msg = self._rc.news[0] if msg.cause_by in write_code_filters: logger.info(f"TODO WriteCode:{msg.json()}") - await self._new_code_actions() + await self._new_code_actions(bug_fix=msg.cause_by == any_to_str(FixBug)) return self._rc.todo if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): logger.info(f"TODO SummarizeCode:{msg.json()}") @@ -232,10 +233,10 @@ async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_r coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) return coding_doc - async def _new_code_actions(self): + async def _new_code_actions(self, bug_fix=False): # Prepare file repos src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + changed_src_files = src_file_repo.all_files if bug_fix else src_file_repo.changed_files task_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) changed_task_files = task_file_repo.changed_files design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 2651be7eb..52ac3cf28 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -286,6 +286,8 @@ async def _act(self) -> Message: cause_by=self._rc.todo, sent_from=self, ) + elif isinstance(response, Message): + msg = response else: msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) diff --git a/metagpt/schema.py b/metagpt/schema.py index a8c1b7726..25281e399 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -97,14 +97,14 @@ class Message(BaseModel): send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) def __init__( - self, - content, - instruct_content=None, - role="user", - cause_by="", - sent_from="", - send_to=MESSAGE_ROUTE_TO_ALL, - **kwargs, + self, + content, + instruct_content=None, + role="user", + cause_by="", + sent_from="", + send_to=MESSAGE_ROUTE_TO_ALL, + **kwargs, ): """ Parameters not listed below will be stored as meta info, including custom parameters. @@ -341,3 +341,7 @@ def loads(filenames: List) -> CodeSummarizeContext: def __hash__(self): return hash((self.design_filename, self.task_filename)) + + +class BugFixContext(BaseModel): + filename: str = "" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 5aec4509c..d372fd22e 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,15 +8,13 @@ """ from __future__ import annotations -import os +from gitignore_parser import parse_gitignore, rule_from_pattern, handle_negation import shutil from enum import Enum from pathlib import Path from typing import Dict, List - from git.repo import Repo from git.repo.fun import is_git_dir - from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile @@ -51,6 +49,7 @@ def __init__(self, local_path=None, auto_init=True): """ self._repository = None self._dependency = None + self._gitignore_rules = None if local_path: self.open(local_path=local_path, auto_init=auto_init) @@ -63,6 +62,7 @@ def open(self, local_path: Path, auto_init=False): local_path = Path(local_path) if self.is_git_dir(local_path): self._repository = Repo(local_path) + self._gitignore_rules = parse_gitignore(full_path=str(local_path / ".gitignore")) return if not auto_init: return @@ -82,6 +82,7 @@ def _init(self, local_path: Path): writer.write("\n".join(ignores)) self._repository.index.add([".gitignore"]) self._repository.index.commit("Add .gitignore") + self._gitignore_rules = parse_gitignore(full_path=gitignore_filename) def add_change(self, files: Dict): """Add or remove files from the staging area based on the provided changes. @@ -204,8 +205,9 @@ def rename_root(self, new_dir_name): logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) - def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None) -> List: - """Retrieve a list of files in the specified relative path. + def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None, filter_ignored=True) -> List: + """ + Retrieve a list of files in the specified relative path. The method returns a list of file paths relative to the current FileRepository. @@ -213,6 +215,8 @@ def get_files(self, relative_path: Path | str, root_relative_path: Path | str = :type relative_path: Path or str :param root_relative_path: The root relative path within the repository. :type root_relative_path: Path or str + :param filter_ignored: Flag to indicate whether to filter files based on .gitignore rules. + :type filter_ignored: bool :return: A list of file paths in the specified directory. :rtype: List[str] """ @@ -231,10 +235,35 @@ def get_files(self, relative_path: Path | str, root_relative_path: Path | str = rpath = file_path.relative_to(root_relative_path) files.append(str(rpath)) else: - subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path) + subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path, + filter_ignored=False) files.extend(subfolder_files) except Exception as e: logger.error(f"Error: {e}") + if not filter_ignored: + return files + filtered_files = self.filter_gitignore(filenames=files, root_relative_path=root_relative_path) + return filtered_files + + def filter_gitignore(self, filenames: List[str], root_relative_path: Path | str = None) -> List[str]: + """ + Filter a list of filenames based on .gitignore rules. + + :param filenames: A list of filenames to be filtered. + :type filenames: List[str] + :param root_relative_path: The root relative path within the repository. + :type root_relative_path: Path or str + :return: A list of filenames that pass the .gitignore filtering. + :rtype: List[str] + """ + if root_relative_path is None: + root_relative_path = self.workdir + files = [] + for filename in filenames: + pathname = root_relative_path / filename + if self._gitignore_rules(str(pathname)): + continue + files.append(filename) return files @@ -244,6 +273,7 @@ def get_files(self, relative_path: Path | str, root_relative_path: Path | str = repo = GitRepository() repo.open(path, auto_init=True) + repo.filter_gitignore(filenames=["snake_game/snake_game/__pycache__", "snake_game/snake_game/game.py"]) changes = repo.changed_files print(changes) diff --git a/requirements.txt b/requirements.txt index 99f738448..515a4d88b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,4 +48,4 @@ websocket-client==0.58.0 aiofiles==23.2.1 gitpython==3.1.40 zhipuai==1.0.7 - +gitignore-parser==0.1.9 diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index 23bebba7f..d800e9594 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -73,6 +73,13 @@ async def test_git1(): repo1 = GitRepository(local_path=local_path, auto_init=False) assert repo1.changed_files + file_repo = repo1.new_file_repository("__pycache__") + await file_repo.save("a.pyc", content="") + all_files = repo1.get_files(relative_path=".", filter_ignored=False) + assert "__pycache__/a.pyc" in all_files + all_files = repo1.get_files(relative_path=".", filter_ignored=True) + assert "__pycache__/a.pyc" not in all_files + repo1.delete_repository() assert not local_path.exists() From 88bbc75d565a8549ed790c78d95fdd6759630085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 13 Dec 2023 22:19:55 +0800 Subject: [PATCH 211/232] fixbug: gitignore error after project renamed --- metagpt/utils/git_repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index d372fd22e..9827b8252 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -204,6 +204,7 @@ def rename_root(self, new_dir_name): logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}") logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) + self._gitignore_rules = parse_gitignore(full_path=str(new_path / ".gitignore")) def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None, filter_ignored=True) -> List: """ From ad0e5a6da83d6ded26f9b7f36b834c0bba78b8b9 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 12 Dec 2023 16:49:41 +0800 Subject: [PATCH 212/232] action_node: make it work at first step. --- metagpt/actions/action_node.py | 258 +++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 metagpt/actions/action_node.py diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py new file mode 100644 index 000000000..4fbd3ce7f --- /dev/null +++ b/metagpt/actions/action_node.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/11 18:45 +@Author : alexanderwu +@File : action_node.py +""" +from typing import Dict, Type, List, Any +import json + +from pydantic import BaseModel, create_model, root_validator, validator +# , model_validator, field_validator + +from metagpt.logs import logger + + +def dict_to_markdown(d, prefix="##", postfix="\n\n"): + markdown_str = "" + for key, value in d.items(): + markdown_str += f"{prefix} {key}: {value}{postfix}" + return markdown_str + + +class ActionNode: + """ActionNode is a tree of nodes.""" + + # Action Inputs + key: str # Product Requirement / File list / Code + expected_type: Type # such as str / int / float etc. + # context: str # everything in the history. + instruction: str # the instructions should be followed. + example: str # example for In Context-Learning. + + # Action Outputs + content: str + instruct_content: BaseModel + children: dict[str, "ActionNode"] + + def __init__(self, key, expected_type, instruction, example, content="", + children=None): + self.key = key + self.expected_type = expected_type + self.instruction = instruction + self.example = example + self.content = content + self.children = children if children is not None else {} + + def __str__(self): + return f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" \ + f", {self.content}, {self.children}" + + def __repr__(self): + return self.__str__() + + def add_child(self, node: "ActionNode"): + """增加子ActionNode""" + self.children[node.key] = node + + def add_childs(self, nodes: List["ActionNode"]): + """批量增加子ActionNode""" + for node in nodes: + self.add_child(node) + + def get_children_mapping(self) -> Dict[str, Type]: + """获得子ActionNode的字典，以key索引""" + return {k: v.expected_type for k, v in self.children.items()} + + @classmethod + def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): + """基于pydantic v1的模型动态生成，用来检验结果类型正确性""" + new_class = create_model(class_name, **mapping) + + @validator("*", allow_reuse=True) + def check_name(v, field): + if field.name not in mapping.keys(): + raise ValueError(f"Unrecognized block: {field.name}") + return v + + @root_validator(pre=True, allow_reuse=True) + def check_missing_fields(values): + required_fields = set(mapping.keys()) + missing_fields = required_fields - set(values.keys()) + if missing_fields: + raise ValueError(f"Missing fields: {missing_fields}") + return values + + new_class.__validator_check_name = classmethod(check_name) + new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields) + return new_class + + @classmethod + def create_model_class_v2(cls, class_name: str, mapping: Dict[str, Type]): + """基于pydantic v2的模型动态生成，用来检验结果类型正确性，待验证""" + new_class = create_model(class_name, **mapping) + + @model_validator(mode='before') + def check_missing_fields(data): + required_fields = set(mapping.keys()) + missing_fields = required_fields - set(data.keys()) + if missing_fields: + raise ValueError(f"Missing fields: {missing_fields}") + return data + + @field_validator('*') + def check_name(v: Any, field: str) -> Any: + if field not in mapping.keys(): + raise ValueError(f"Unrecognized block: {field}") + return v + + new_class.__model_validator_check_missing_fields = classmethod(check_missing_fields) + new_class.__field_validator_check_name = classmethod(check_name) + return new_class + + def create_children_class(self): + """使用object内有的字段直接生成model_class""" + class_name = f"{self.key}_AN" + mapping = self.get_children_mapping() + return self.create_model_class(class_name, mapping) + + def to_dict(self, format_func=None, mode="all") -> Dict: + # 如果没有提供格式化函数，使用默认的格式化方式 + if format_func is None: + format_func = lambda node: f"{node.instruction}" + + # 使用提供的格式化函数来格式化当前节点的值 + formatted_value = format_func(self) + + # 创建当前节点的键值对 + if mode == "children": + node_dict = {} + else: + node_dict = {self.key: formatted_value} + + if mode == "root": + return node_dict + + # 遍历子节点并递归调用 to_dict 方法 + for child_key, child_node in self.children.items(): + node_dict.update(child_node.to_dict(format_func)) + + return node_dict + + def compile_to(self, i: Dict, to="raw") -> str: + if to == "json": + return json.dumps(i, indent=4) + elif to == "markdown": + return dict_to_markdown(i) + else: + return str(i) + + def compile_instruction(self, to="raw", mode="children") -> str: + """compile to raw/json/markdown template with all/root/children nodes""" + format_func = lambda i: f"{i.expected_type} # {i.instruction}" + nodes = self.to_dict(format_func=format_func, mode=mode) + return self.compile_to(nodes, to) + + def compile_example(self, to="raw", mode="all") -> str: + """compile to raw/json/markdown examples with all/root/children nodes""" + format_func = lambda i: f"{i.example}" + nodes = self.to_dict(format_func=format_func, mode=mode) + return self.compile_to(nodes, to) + + def compile(self, to="raw", mode="all") -> str: + pass + + def run(self): + """运行这个ActionNode，可以采用不同策略，比如只运行子节点""" + pass + + +IMPLEMENTATION_APPROACH = ActionNode( + key="implementation_approach", + expected_type=str, + instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework", + example="We will ..." +) + +PROJECT_NAME = ActionNode( + key="project_name", + expected_type=str, + instruction="The project name with underline", + example="game_2048" +) + +FILE_LIST = ActionNode( + key="file_list", + expected_type=List[str], + instruction="Only need relative paths. ALWAYS write a main.py or app.py here", + example="['main.py', 'const.py', 'utils.py']" +) + +DATA_STRUCTURES_AND_INTERFACES = ActionNode( + key="data_structures_and_interfaces", + expected_type=str, + instruction="Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions " + "(with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " + "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", + example="""classDiagram +class Game{{ + +int score +}} +... +Game "1" -- "1" Food: has""" +) + +PROGRAM_CALL_FLOW = ActionNode( + key="program_call_flow", + expected_type=str, + instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE " + "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", + example="""sequenceDiagram +participant M as Main +... +G->>M: end game""" +) + +ANYTHING_UNCLEAR = ActionNode( + key="anything_unclear", + expected_type=str, + instruction="Mention unclear project aspects, then try to clarify it.", + example="Clarification needed on third-party API integration, ..." +) + + +ACTION_NODES = [ + IMPLEMENTATION_APPROACH, + PROJECT_NAME, + FILE_LIST, + DATA_STRUCTURES_AND_INTERFACES, + PROGRAM_CALL_FLOW, + ANYTHING_UNCLEAR +] + + +def action_node_from_tuple_example(): + # 示例：列表中包含元组 + list_of_tuples = [ + ("key1", str, "Instruction 1", "Example 1", "Content 1", {"child1": ...}), + ("key2", int, "Instruction 2", "Example 2", "Content 2"), + ("key3", int, "Instruction 3", "Example 3") + ] + + # 从列表中创建 ActionNode 实例 + nodes = [ActionNode(*data) for data in list_of_tuples] + for i in nodes: + logger.info(i) + + +def main(): + write_design_node = ActionNode("WriteDesign", str, "", "") + write_design_node.add_childs(ACTION_NODES) + instruction = write_design_node.compile_instruction(to="markdown") + logger.info(instruction) + logger.info(write_design_node.compile_example()) + + +if __name__ == '__main__': + main() From bfdb8415adc0c23ef7654402c862bf8302d34f92 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 13 Dec 2023 17:47:09 +0800 Subject: [PATCH 213/232] tuning action node code --- metagpt/actions/action.py | 6 +- metagpt/actions/action_node.py | 142 ++++++++++++------------------- metagpt/actions/design_api.py | 12 ++- metagpt/actions/write_prd.py | 4 +- metagpt/config.py | 2 +- metagpt/environment.py | 2 +- metagpt/utils/file_repository.py | 2 +- 7 files changed, 73 insertions(+), 97 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index dc96699a9..40faaad41 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -25,9 +25,9 @@ def __init__(self, name: str = "", context=None, llm: LLM = None): llm = LLM() self.llm = llm self.context = context - self.prefix = "" - self.profile = "" - self.desc = "" + self.prefix = "" # aask*时会加上prefix，作为system_message + self.profile = "" # FIXME: USELESS + self.desc = "" # FIXME: USELESS self.content = "" self.instruct_content = None self.env = None diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 4fbd3ce7f..35912446d 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : action_node.py """ -from typing import Dict, Type, List, Any +from typing import Dict, Type, List, Any, Tuple import json from pydantic import BaseModel, create_model, root_validator, validator @@ -14,7 +14,16 @@ from metagpt.logs import logger -def dict_to_markdown(d, prefix="##", postfix="\n\n"): +SIMPLE_TEMPLATE = """ +## example +{example} + +## instruction +{instruction} +""" + + +def dict_to_markdown(d, prefix="###", postfix="\n"): markdown_str = "" for key, value in d.items(): markdown_str += f"{prefix} {key}: {value}{postfix}" @@ -23,13 +32,17 @@ def dict_to_markdown(d, prefix="##", postfix="\n\n"): class ActionNode: """ActionNode is a tree of nodes.""" + # 应该是定义子任务，收集子任务结果，并且父任务同时执行吗？ + # 初期只提供两种模式，一种是用父任务compile，一种是用子任务逐个执行 + # 1. context、example、instruction-nodes、instruction-action + # 2. context、example # Action Inputs key: str # Product Requirement / File list / Code expected_type: Type # such as str / int / float etc. # context: str # everything in the history. instruction: str # the instructions should be followed. - example: str # example for In Context-Learning. + example: Any # example for In Context-Learning. # Action Outputs content: str @@ -56,7 +69,7 @@ def add_child(self, node: "ActionNode"): """增加子ActionNode""" self.children[node.key] = node - def add_childs(self, nodes: List["ActionNode"]): + def add_children(self, nodes: List["ActionNode"]): """批量增加子ActionNode""" for node in nodes: self.add_child(node) @@ -140,7 +153,7 @@ def to_dict(self, format_func=None, mode="all") -> Dict: return node_dict - def compile_to(self, i: Dict, to="raw") -> str: + def compile_to(self, i: Dict, to) -> str: if to == "json": return json.dumps(i, indent=4) elif to == "markdown": @@ -148,88 +161,49 @@ def compile_to(self, i: Dict, to="raw") -> str: else: return str(i) - def compile_instruction(self, to="raw", mode="children") -> str: + def tagging(self, text, to, tag="") -> str: + if not tag: + return text + if to == "json": + return f"[{tag}]\n" + "{" + text + "}" + f"\n[/{tag}]" + else: + return f"[{tag}]\n" + text + f"\n[/{tag}]" + + def _compile_f(self, to, mode, tag, format_func) -> str: + nodes = self.to_dict(format_func=format_func, mode=mode) + text = self.compile_to(nodes, to) + return self.tagging(text, to, tag) + + def compile_instruction(self, to="raw", mode="children", tag="") -> str: """compile to raw/json/markdown template with all/root/children nodes""" format_func = lambda i: f"{i.expected_type} # {i.instruction}" - nodes = self.to_dict(format_func=format_func, mode=mode) - return self.compile_to(nodes, to) + return self._compile_f(to, mode, tag, format_func) - def compile_example(self, to="raw", mode="all") -> str: + def compile_example(self, to="raw", mode="children", tag="") -> str: """compile to raw/json/markdown examples with all/root/children nodes""" - format_func = lambda i: f"{i.example}" - nodes = self.to_dict(format_func=format_func, mode=mode) - return self.compile_to(nodes, to) - def compile(self, to="raw", mode="all") -> str: - pass + # 这里不能使用f-string，因为转译为str后再json.dumps会额外加上引号，无法作为有效的example + # 错误示例："File list": "['main.py', 'const.py', 'game.py']", 注意这里值不是list，而是str + format_func = lambda i: i.example + return self._compile_f(to, mode, tag, format_func) + + def compile(self, mode="children") -> Tuple[str, str]: + """ + mode: all/root/children + mode="children": 编译所有子节点为一个统一模板，包括instruction与example + mode="all": NotImplemented + mode="root": NotImplemented + """ + self.instruction = self.compile_instruction(to="json", mode=mode) + self.example = self.compile_example(to="json", tag="CONTENT", mode=mode) + # prompt = template.format(example=self.example, instruction=self.instruction) + return self.instruction, self.example def run(self): """运行这个ActionNode，可以采用不同策略，比如只运行子节点""" - pass - - -IMPLEMENTATION_APPROACH = ActionNode( - key="implementation_approach", - expected_type=str, - instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework", - example="We will ..." -) - -PROJECT_NAME = ActionNode( - key="project_name", - expected_type=str, - instruction="The project name with underline", - example="game_2048" -) - -FILE_LIST = ActionNode( - key="file_list", - expected_type=List[str], - instruction="Only need relative paths. ALWAYS write a main.py or app.py here", - example="['main.py', 'const.py', 'utils.py']" -) - -DATA_STRUCTURES_AND_INTERFACES = ActionNode( - key="data_structures_and_interfaces", - expected_type=str, - instruction="Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions " - "(with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " - "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", - example="""classDiagram -class Game{{ - +int score -}} -... -Game "1" -- "1" Food: has""" -) - -PROGRAM_CALL_FLOW = ActionNode( - key="program_call_flow", - expected_type=str, - instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE " - "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", - example="""sequenceDiagram -participant M as Main -... -G->>M: end game""" -) - -ANYTHING_UNCLEAR = ActionNode( - key="anything_unclear", - expected_type=str, - instruction="Mention unclear project aspects, then try to clarify it.", - example="Clarification needed on third-party API integration, ..." -) - - -ACTION_NODES = [ - IMPLEMENTATION_APPROACH, - PROJECT_NAME, - FILE_LIST, - DATA_STRUCTURES_AND_INTERFACES, - PROGRAM_CALL_FLOW, - ANYTHING_UNCLEAR -] + + # 需要传入llm，并且实际在ActionNode中执行。需要规划好具体的执行方法 + raise NotImplementedError def action_node_from_tuple_example(): @@ -246,13 +220,5 @@ def action_node_from_tuple_example(): logger.info(i) -def main(): - write_design_node = ActionNode("WriteDesign", str, "", "") - write_design_node.add_childs(ACTION_NODES) - instruction = write_design_node.compile_instruction(to="markdown") - logger.info(instruction) - logger.info(write_design_node.compile_example()) - - if __name__ == '__main__': - main() + action_node_from_tuple_example() diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 557ebcbbd..a6d559a4c 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -14,6 +14,7 @@ from typing import List from metagpt.actions import Action, ActionOutput +from metagpt.actions.design_api_an import DESIGN_API_NODE, SIMPLE_TEMPLATE from metagpt.config import CONFIG from metagpt.const import ( DATA_API_DESIGN_FILE_REPO, @@ -227,13 +228,22 @@ async def run(self, with_messages, format=CONFIG.prompt_format): # leaving room for global optimization in subsequent steps. return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - async def _new_system_design(self, context, format=CONFIG.prompt_format): + async def _new_system_design_bakup(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) return system_design + async def _new_system_design(self, context, format=CONFIG.prompt_format): + instruction, example = DESIGN_API_NODE.compile() + prompt = SIMPLE_TEMPLATE.format(context=context, example=example, instruction=instruction) + # prompt_template, format_example = get_template(templates, format) + # format_example = format_example.format(project_name=CONFIG.project_name) + # prompt = prompt_template.format(context=context, format_example=format_example) + system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) + return system_design + async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): prompt = MERGE_PROMPT.format( old_design=system_design_doc.content, context=prd_doc.content, project_name=CONFIG.project_name diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index aad2422ef..0594d116e 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -361,7 +361,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc - logger.info(f"NEW PRD:{prd_doc.filename}") + logger.debug(f"new prd: {prd_doc.filename}") # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the # 'publish' message to transition the workflow to the next stage. This design allows room for global # optimization in subsequent steps. @@ -394,7 +394,7 @@ async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) res = await self._aask(prompt=prompt) - logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + logger.info(f"REQ-RELATIVE: [{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") if "YES" in res: return True return False diff --git a/metagpt/config.py b/metagpt/config.py index d04ae7291..d2390f704 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -108,7 +108,7 @@ def _update(self): def _ensure_workspace_exists(self): self.workspace_path.mkdir(parents=True, exist_ok=True) - logger.info(f"WORKSPACE_PATH set to {self.workspace_path}") + logger.debug(f"WORKSPACE_PATH set to {self.workspace_path}") def _init_with_config_files_and_env(self, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/environment.py b/metagpt/environment.py index 02eb3d340..7d1e307f3 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -58,7 +58,7 @@ def publish_message(self, message: Message) -> bool: route the message to the message recipient is a problem addressed by the transport framework designed in RFC 113. """ - logger.info(f"publish_message: {message.dump()}") + logger.debug(f"publish_message: {message.dump()}") found = False # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113 for role, subscription in self.members.items(): diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 2cace7232..2eca799a8 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -205,7 +205,7 @@ async def save_doc(self, doc: Document, with_suffix: str = None, dependencies: L m = json.loads(doc.content) filename = Path(doc.filename).with_suffix(with_suffix) if with_suffix is not None else Path(doc.filename) await self.save(filename=str(filename), content=json_to_markdown(m), dependencies=dependencies) - logger.info(f"File Saved: {str(filename)}") + logger.debug(f"File Saved: {str(filename)}") @staticmethod async def get_file(filename: Path | str, relative_path: Path | str = ".") -> Document | None: From 5d7c228539be3d50e1e97d8927cef34852117f82 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 13 Dec 2023 17:47:19 +0800 Subject: [PATCH 214/232] tuning action node code --- metagpt/actions/design_api_an.py | 146 +++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 metagpt/actions/design_api_an.py diff --git a/metagpt/actions/design_api_an.py b/metagpt/actions/design_api_an.py new file mode 100644 index 000000000..b4bd54849 --- /dev/null +++ b/metagpt/actions/design_api_an.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/12 22:24 +@Author : alexanderwu +@File : design_api_an.py +""" +from metagpt.actions.action_node import ActionNode +from metagpt.logs import logger + +IMPLEMENTATION_APPROACH = ActionNode( + key="Implementation approach", + expected_type=str, + instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework", + example="We will ..." +) + +PROJECT_NAME = ActionNode( + key="Project name", + expected_type=str, + instruction="The project name with underline", + example="game_2048" +) + +FILE_LIST = ActionNode( + key="File list", + expected_type=list[str], + instruction="Only need relative paths. ALWAYS write a main.py or app.py here", + example=['main.py', 'game.py'] +) + +DATA_STRUCTURES_AND_INTERFACES = ActionNode( + key="Data structures and interfaces", + expected_type=str, + instruction="Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions " + "(with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " + "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", + example=""" classDiagram + class User { + +int id + +str username + +str email + +str password + __init__(id: int, username: str, email: str, password: str) + follow(user: User): void + like(content: Content): void + comment(content: Content, text: str): Comment + } + class Content { + +int id + +User author + +str title + +str body + +datetime created_at + +list likes + +list comments + __init__(id: int, author: User, title: str, body: str) + get_likes(): list + get_comments(): list + } + class Comment { + +int id + +User author + +str text + +datetime created_at + __init__(id: int, author: User, text: str) + } + class Leaderboard { + +list top_contents + update(): void + } + class SearchEngine { + +str query + search(): list + } + class RecommendationEngine { + +User user + recommend(): list + } + class TaskQueue { + +str task_name + enqueue(task: function): void + } + User "1" -- "*" Content: creates + Content "1" -- "*" Comment: includes + User "1" -- "*" Comment: writes + User "1" -- "*" User: follows + Content "1" -- "*" User: liked_by""" +) + +PROGRAM_CALL_FLOW = ActionNode( + key="Program call flow", + expected_type=str, + instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE " + "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", + example="""sequenceDiagram +participant M as Main +... +G->>M: end game""" +) + +ANYTHING_UNCLEAR = ActionNode( + key="Anything UNCLEAR", + expected_type=str, + instruction="Mention unclear project aspects, then try to clarify it.", + example="Clarification needed on third-party API integration, ..." +) + +ACTION_NODES = [ + IMPLEMENTATION_APPROACH, + PROJECT_NAME, + FILE_LIST, + DATA_STRUCTURES_AND_INTERFACES, + PROGRAM_CALL_FLOW, + ANYTHING_UNCLEAR +] + +DESIGN_API_NODE = ActionNode("DesignAPI", str, "", "") +DESIGN_API_NODE.add_children(ACTION_NODES) + +SIMPLE_TEMPLATE = """ +## context +{context} + +## example +{example} + +## instruction-nodes: ": # " +{instruction} + +## instruction-action +Role: You are an architect; the goal is to design a SOTA software system +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +Requirement: Fill in the above missing instruction-nodes based on the context +now, output wrapped inside [CONTENT][/CONTENT] as example, nothing else. +""" + + +def main(): + instruction, example = DESIGN_API_NODE.compile() + text = SIMPLE_TEMPLATE.format(context="", example=example, instruction=instruction) + logger.info(text) + + +if __name__ == '__main__': + main() From c0bcf57caf134008ea5c8bd9a2df3cbdb3465759 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 14 Dec 2023 15:58:05 +0800 Subject: [PATCH 215/232] Transfer Action usage to ActionNode for subsequent structured reasoning opportunities - Modifided actions: project_management / design_api / write_prd --- metagpt/actions/action.py | 20 +- metagpt/actions/action_node.py | 122 ++++++-- metagpt/actions/design_api.py | 189 +----------- metagpt/actions/design_api_an.py | 91 +----- metagpt/actions/project_management.py | 206 +------------ metagpt/actions/project_management_an.py | 82 +++++ metagpt/actions/write_prd.py | 366 +++-------------------- metagpt/actions/write_prd_an.py | 153 ++++++++++ metagpt/environment.py | 2 +- metagpt/llm.py | 3 +- metagpt/roles/architect.py | 4 +- metagpt/roles/engineer.py | 9 +- metagpt/roles/project_manager.py | 3 +- metagpt/roles/role.py | 5 +- metagpt/roles/searcher.py | 3 +- 15 files changed, 438 insertions(+), 820 deletions(-) create mode 100644 metagpt/actions/project_management_an.py create mode 100644 metagpt/actions/write_prd_an.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 40faaad41..2fd130cf5 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -27,18 +27,22 @@ def __init__(self, name: str = "", context=None, llm: LLM = None): self.context = context self.prefix = "" # aask*时会加上prefix，作为system_message self.profile = "" # FIXME: USELESS - self.desc = "" # FIXME: USELESS - self.content = "" - self.instruct_content = None - self.env = None + self.desc = "" # for skill manager + self.nodes = ... - def set_env(self, env): - self.env = env + # Output, useless + # self.content = "" + # self.instruct_content = None + # self.env = None + + # def set_env(self, env): + # self.env = env def set_prefix(self, prefix, profile): """Set prefix for later usage""" self.prefix = prefix self.profile = profile + self.llm.system_prompt = prefix def __str__(self): return self.__class__.__name__ @@ -62,10 +66,6 @@ async def _aask_v1( system_msgs: Optional[list[str]] = None, format="markdown", # compatible to original format ) -> ActionOutput: - """Append default prefix""" - if not system_msgs: - system_msgs = [] - system_msgs.append(self.prefix) content = await self.llm.aask(prompt, system_msgs) logger.debug(content) output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 35912446d..178986ebe 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -5,25 +5,44 @@ @Author : alexanderwu @File : action_node.py """ -from typing import Dict, Type, List, Any, Tuple +import re +from typing import Dict, Type, List, Any, Tuple, Optional import json from pydantic import BaseModel, create_model, root_validator, validator # , model_validator, field_validator +from tenacity import wait_random_exponential, stop_after_attempt, retry +from metagpt.actions import ActionOutput +from metagpt.llm import BaseGPTAPI from metagpt.logs import logger +from metagpt.utils.common import OutputParser +from metagpt.utils.custom_decoder import CustomDecoder +CONSTRAINT = """ +- Language: Please use the same language as the user input. +- Format: output wrapped inside [CONTENT][/CONTENT] as format example, nothing else. +""" SIMPLE_TEMPLATE = """ -## example +## context +{context} + +## format example {example} -## instruction +## nodes: ": # " {instruction} + +## constraint +{constraint} + +## action +Fill in the above nodes based on the context. Answer in format example. """ -def dict_to_markdown(d, prefix="###", postfix="\n"): +def dict_to_markdown(d, prefix="-", postfix="\n"): markdown_str = "" for key, value in d.items(): markdown_str += f"{prefix} {key}: {value}{postfix}" @@ -32,22 +51,26 @@ def dict_to_markdown(d, prefix="###", postfix="\n"): class ActionNode: """ActionNode is a tree of nodes.""" - # 应该是定义子任务，收集子任务结果，并且父任务同时执行吗？ - # 初期只提供两种模式，一种是用父任务compile，一种是用子任务逐个执行 - # 1. context、example、instruction-nodes、instruction-action - # 2. context、example + # Action Strgy + # - sop: 仅使用一级SOP + # - complex: 使用一级SOP+自定义策略填槽 + mode: str + + # Action Context + context: str # all the context, including all necessary info + llm: BaseGPTAPI # LLM with aask interface + children: dict[str, "ActionNode"] - # Action Inputs + # Action Input key: str # Product Requirement / File list / Code expected_type: Type # such as str / int / float etc. # context: str # everything in the history. instruction: str # the instructions should be followed. example: Any # example for In Context-Learning. - # Action Outputs + # Action Output content: str instruct_content: BaseModel - children: dict[str, "ActionNode"] def __init__(self, key, expected_type, instruction, example, content="", children=None): @@ -74,9 +97,16 @@ def add_children(self, nodes: List["ActionNode"]): for node in nodes: self.add_child(node) + @classmethod + def from_children(cls, key, nodes: List["ActionNode"]): + """直接从一系列的子nodes初始化""" + obj = cls(key, str, "", "") + obj.add_children(nodes) + return obj + def get_children_mapping(self) -> Dict[str, Type]: """获得子ActionNode的字典，以key索引""" - return {k: v.expected_type for k, v in self.children.items()} + return {k: (v.expected_type, ...) for k, v in self.children.items()} @classmethod def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): @@ -131,6 +161,8 @@ def create_children_class(self): return self.create_model_class(class_name, mapping) def to_dict(self, format_func=None, mode="all") -> Dict: + """将当前节点与子节点都按照node: format的格式组织称字典""" + # 如果没有提供格式化函数，使用默认的格式化方式 if format_func is None: format_func = lambda node: f"{node.instruction}" @@ -165,7 +197,7 @@ def tagging(self, text, to, tag="") -> str: if not tag: return text if to == "json": - return f"[{tag}]\n" + "{" + text + "}" + f"\n[/{tag}]" + return f"[{tag}]\n" + text + f"\n[/{tag}]" else: return f"[{tag}]\n" + text + f"\n[/{tag}]" @@ -187,31 +219,73 @@ def compile_example(self, to="raw", mode="children", tag="") -> str: format_func = lambda i: i.example return self._compile_f(to, mode, tag, format_func) - def compile(self, mode="children") -> Tuple[str, str]: + def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) -> str: """ mode: all/root/children mode="children": 编译所有子节点为一个统一模板，包括instruction与example mode="all": NotImplemented mode="root": NotImplemented """ - self.instruction = self.compile_instruction(to="json", mode=mode) - self.example = self.compile_example(to="json", tag="CONTENT", mode=mode) - # prompt = template.format(example=self.example, instruction=self.instruction) - return self.instruction, self.example - def run(self): - """运行这个ActionNode，可以采用不同策略，比如只运行子节点""" + # FIXME: json instruction会带来 "Project name": "web_2048 # 项目名称使用下划线", + self.instruction = self.compile_instruction(to="markdown", mode=mode) + self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) + prompt = template.format(context=context, example=self.example, instruction=self.instruction, + constraint=CONSTRAINT) + return prompt + + @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) + async def _aask_v1( + self, + prompt: str, + output_class_name: str, + output_data_mapping: dict, + system_msgs: Optional[list[str]] = None, + format="markdown", # compatible to original format + ) -> ActionOutput: + content = await self.llm.aask(prompt, system_msgs) + logger.debug(content) + output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) + + if format == "json": + pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" + matches = re.findall(pattern, content, re.DOTALL) + + for match in matches: + if match: + content = match + break + + parsed_data = CustomDecoder(strict=False).decode(content) + + else: # using markdown parser + parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) + + logger.debug(parsed_data) + instruct_content = output_class(**parsed_data) + return ActionOutput(content, instruct_content) + + def get(self, key): + return self.instruct_content.dict()[key] + + async def fill(self, context, llm, to="json"): + """运行这个ActionNode，并且填槽，可以采用不同策略，比如只运行子节点""" + self.llm = llm + prompt = self.compile(context=context, to=to) + mapping = self.get_children_mapping() + class_name = f"{self.key}_AN" # 需要传入llm，并且实际在ActionNode中执行。需要规划好具体的执行方法 - raise NotImplementedError + output = await self._aask_v1(prompt, class_name, mapping, format=to) + self.content = output.content + self.instruct_content = output.instruct_content + return self def action_node_from_tuple_example(): # 示例：列表中包含元组 list_of_tuples = [ - ("key1", str, "Instruction 1", "Example 1", "Content 1", {"child1": ...}), - ("key2", int, "Instruction 2", "Example 2", "Content 2"), - ("key3", int, "Instruction 3", "Example 3") + ("key1", int, "Instruction 1", "Example 1") ] # 从列表中创建 ActionNode 实例 diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index a6d559a4c..fd58e0ca8 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,10 +11,10 @@ """ import json from pathlib import Path -from typing import List +# from typing import List from metagpt.actions import Action, ActionOutput -from metagpt.actions.design_api_an import DESIGN_API_NODE, SIMPLE_TEMPLATE +from metagpt.actions.design_api_an import DESIGN_API_NODE from metagpt.config import CONFIG from metagpt.const import ( DATA_API_DESIGN_FILE_REPO, @@ -26,166 +26,15 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository -from metagpt.utils.get_template import get_template +# from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file -templates = { - "json": { - "PROMPT_TEMPLATE": """ -# Context -{context} - -## Format example -{format_example} ------ -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system -Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirement: Fill in the following missing information based on the context, each section name is a key in json - -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. - -## Project name: Constant text. - -## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here - -## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. - -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, -and only output the json inside this tag, nothing else -""", - "FORMAT_EXAMPLE": """ -[CONTENT] -{{ - "Implementation approach": "We will ...", - "Project name": "{project_name}", - "File list": ["main.py"], - "Data structures and interfaces": ' - classDiagram - class Game{{ - +int score - }} - ... - Game "1" -- "1" Food: has - ', - "Program call flow": ' - sequenceDiagram - participant M as Main - ... - G->>M: end game - ', - "Anything UNCLEAR": "The requirement is clear to me." -}} -[/CONTENT] -""", - }, - "markdown": { - "PROMPT_TEMPLATE": """ -# Context -{context} - -## Format example -{format_example} ------ -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools -Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -ATTENTION: Output carefully referenced "Format example" in format. - -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. - -## Project name: Constant text. - -## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here - -## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. - -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. - -""", - "FORMAT_EXAMPLE": """ ---- -## Implementation approach -We will ... - -## Project name -```python -"{project_name}" -``` - -## File list -```python -[ - "main.py", -] -``` - -## Data structures and interfaces -```mermaid -classDiagram - class Game{ - +int score - } - ... - Game "1" -- "1" Food: has -``` - -## Program call flow -```mermaid -sequenceDiagram - participant M as Main - ... - G->>M: end game -``` - -## Anything UNCLEAR -The requirement is clear to me. ---- -""", - }, -} - -OUTPUT_MAPPING = { - "Implementation approach": (str, ...), - "Project name": (str, ...), - "File list": (List[str], ...), - "Data structures and interfaces": (str, ...), - "Program call flow": (str, ...), - "Anything UNCLEAR": (str, ...), -} - -MERGE_PROMPT = """ -## Old Design +NEW_REQ_TEMPLATE = """ +### Legacy Content {old_design} -## Context +### New Requirements {context} - ------ -Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a SOTA PEP8-compliant python system; make the best use of good open source tools -Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -ATTENTION: Output carefully referenced "Old Design" in format. - -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. - -## Project name: Constant text "{project_name}". - -## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here - -## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. - -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, -and only output the json inside this tag, nothing else """ @@ -228,30 +77,16 @@ async def run(self, with_messages, format=CONFIG.prompt_format): # leaving room for global optimization in subsequent steps. return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - async def _new_system_design_bakup(self, context, format=CONFIG.prompt_format): - prompt_template, format_example = get_template(templates, format) - format_example = format_example.format(project_name=CONFIG.project_name) - prompt = prompt_template.format(context=context, format_example=format_example) - system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - return system_design - async def _new_system_design(self, context, format=CONFIG.prompt_format): - instruction, example = DESIGN_API_NODE.compile() - prompt = SIMPLE_TEMPLATE.format(context=context, example=example, instruction=instruction) - # prompt_template, format_example = get_template(templates, format) - # format_example = format_example.format(project_name=CONFIG.project_name) - # prompt = prompt_template.format(context=context, format_example=format_example) - system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - return system_design + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format) + return node async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - prompt = MERGE_PROMPT.format( - old_design=system_design_doc.content, context=prd_doc.content, project_name=CONFIG.project_name + context = NEW_REQ_TEMPLATE.format( + old_design=system_design_doc.content, context=prd_doc.content ) - system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python - # package name" contain space, have to use setattr - system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format) + system_design_doc.content = node.instruct_content.json(ensure_ascii=False) return system_design_doc async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: diff --git a/metagpt/actions/design_api_an.py b/metagpt/actions/design_api_an.py index b4bd54849..2db203606 100644 --- a/metagpt/actions/design_api_an.py +++ b/metagpt/actions/design_api_an.py @@ -6,6 +6,7 @@ @File : design_api_an.py """ from metagpt.actions.action_node import ActionNode +from metagpt.utils.mermaid import MMC1, MMC2 from metagpt.logs import logger IMPLEMENTATION_APPROACH = ActionNode( @@ -32,60 +33,10 @@ DATA_STRUCTURES_AND_INTERFACES = ActionNode( key="Data structures and interfaces", expected_type=str, - instruction="Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions " - "(with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " + instruction="Use mermaid classDiagram code syntax, including classes, method(__init__ etc.) and functions with type" + " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", - example=""" classDiagram - class User { - +int id - +str username - +str email - +str password - __init__(id: int, username: str, email: str, password: str) - follow(user: User): void - like(content: Content): void - comment(content: Content, text: str): Comment - } - class Content { - +int id - +User author - +str title - +str body - +datetime created_at - +list likes - +list comments - __init__(id: int, author: User, title: str, body: str) - get_likes(): list - get_comments(): list - } - class Comment { - +int id - +User author - +str text - +datetime created_at - __init__(id: int, author: User, text: str) - } - class Leaderboard { - +list top_contents - update(): void - } - class SearchEngine { - +str query - search(): list - } - class RecommendationEngine { - +User user - recommend(): list - } - class TaskQueue { - +str task_name - enqueue(task: function): void - } - User "1" -- "*" Content: creates - Content "1" -- "*" Comment: includes - User "1" -- "*" Comment: writes - User "1" -- "*" User: follows - Content "1" -- "*" User: liked_by""" + example=MMC1 ) PROGRAM_CALL_FLOW = ActionNode( @@ -93,10 +44,7 @@ class TaskQueue { expected_type=str, instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE " "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", - example="""sequenceDiagram -participant M as Main -... -G->>M: end game""" + example=MMC2 ) ANYTHING_UNCLEAR = ActionNode( @@ -106,40 +54,21 @@ class TaskQueue { example="Clarification needed on third-party API integration, ..." ) -ACTION_NODES = [ +NODES = [ IMPLEMENTATION_APPROACH, - PROJECT_NAME, + # PROJECT_NAME, FILE_LIST, DATA_STRUCTURES_AND_INTERFACES, PROGRAM_CALL_FLOW, ANYTHING_UNCLEAR ] -DESIGN_API_NODE = ActionNode("DesignAPI", str, "", "") -DESIGN_API_NODE.add_children(ACTION_NODES) - -SIMPLE_TEMPLATE = """ -## context -{context} - -## example -{example} - -## instruction-nodes: ": # " -{instruction} - -## instruction-action -Role: You are an architect; the goal is to design a SOTA software system -Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirement: Fill in the above missing instruction-nodes based on the context -now, output wrapped inside [CONTENT][/CONTENT] as example, nothing else. -""" +DESIGN_API_NODE = ActionNode.from_children("DesignAPI", NODES) def main(): - instruction, example = DESIGN_API_NODE.compile() - text = SIMPLE_TEMPLATE.format(context="", example=example, instruction=instruction) - logger.info(text) + prompt = DESIGN_API_NODE.compile(context="") + logger.info(prompt) if __name__ == '__main__': diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 95da0d65a..29e3bed3e 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -10,10 +10,11 @@ 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ import json -from typing import List +# from typing import List from metagpt.actions import ActionOutput from metagpt.actions.action import Action +from metagpt.actions.project_management_an import PM_NODE from metagpt.config import CONFIG from metagpt.const import ( PACKAGE_REQUIREMENTS_FILENAME, @@ -24,189 +25,14 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository -from metagpt.utils.get_template import get_template +# from metagpt.utils.get_template import get_template -templates = { - "json": { - "PROMPT_TEMPLATE": """ -# Context -{context} - -## Format example -{format_example} ------ -Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules -Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them -ATTENTION: Output carefully referenced "Format example" in format. - -## Required Python third-party packages: Provide Python list[str] in requirements.txt format - -## Required Other language third-party packages: Provide Python list[str] in requirements.txt format - -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first - -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, -and only output the json inside this tag, nothing else -""", - "FORMAT_EXAMPLE": ''' -{ - "Required Python third-party packages": [ - "flask==1.1.2", - "bcrypt==3.2.0" - ], - "Required Other language third-party packages": [ - "No third-party ..." - ], - "Logic Analysis": [ - ["game.py", "Contains..."] - ], - "Task list": [ - "game.py" - ], - "Full API spec": """ - openapi: 3.0.0 - ... - description: A JSON object ... - """, - "Shared Knowledge": """ - 'game.py' contains ... - """, - "Anything UNCLEAR": "We need ... how to start." -} -''', - }, - "markdown": { - "PROMPT_TEMPLATE": """ -# Context -{context} - -## Format example -{format_example} ------ -Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules -Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## Required Python third-party packages: Provided in requirements.txt format - -## Required Other language third-party packages: Provided in requirements.txt format - -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first - -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. - -""", - "FORMAT_EXAMPLE": ''' ---- -## Required Python third-party packages -```python -""" -flask==1.1.2 -bcrypt==3.2.0 -""" -``` - -## Required Other language third-party packages -```python -""" -No third-party ... -""" -``` - -## Full API spec -```python -""" -openapi: 3.0.0 -... -description: A JSON object ... -""" -``` - -## Logic Analysis -```python -[ - ["index.js", "Contains ..."], - ["main.py", "Contains ..."], -] -``` - -## Task list -```python -[ - "index.js", - "main.py", -] -``` - -## Shared Knowledge -```python -""" -'game.py' contains ... -""" -``` - -## Anything UNCLEAR -We need ... how to start. ---- -''', - }, -} -OUTPUT_MAPPING = { - "Required Python third-party packages": (List[str], ...), - "Required Other language third-party packages": (List[str], ...), - "Full API spec": (str, ...), - "Logic Analysis": (List[List[str]], ...), - "Task list": (List[str], ...), - "Shared Knowledge": (str, ...), - "Anything UNCLEAR": (str, ...), -} - -MERGE_PROMPT = """ -# Context -{context} - -## Old Tasks +NEW_REQ_TEMPLATE = """ +### Legacy Content {old_tasks} ------ - -## Format example -{format_example} ------ -Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. -Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## Required Python third-party packages: Provided in requirements.txt format - -## Required Other language third-party packages: Provided in requirements.txt format - -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format, -and only output the json inside this tag, nothing else +### New Requirements +{context} """ @@ -262,18 +88,16 @@ async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo return task_doc async def _run_new_tasks(self, context, format=CONFIG.prompt_format): - prompt_template, format_example = get_template(templates, format) - prompt = prompt_template.format(context=context, format_example=format_example) - rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - return rsp + node = await PM_NODE.fill(context, self.llm, format) + # prompt_template, format_example = get_template(templates, format) + # prompt = prompt_template.format(context=context, format_example=format_example) + # rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) + return node async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: - _, format_example = get_template(templates, format) - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, - format_example=format_example) - rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - task_doc.content = rsp.instruct_content.json(ensure_ascii=False) - return task_doc + context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content) + node = await PM_NODE.fill(context, self.llm, format) + return node @staticmethod async def _update_requirements(doc): diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py new file mode 100644 index 000000000..aa7cdcde2 --- /dev/null +++ b/metagpt/actions/project_management_an.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/14 15:28 +@Author : alexanderwu +@File : project_management_an.py +""" +from metagpt.actions.action_node import ActionNode +from metagpt.logs import logger + +REQUIRED_PYTHON_PACKAGES = ActionNode( + key="Required Python packages", + expected_type=list[str], + instruction="Provide required Python packages in requirements.txt format.", + example=["flask==1.1.2", "bcrypt==3.2.0"] +) + +REQUIRED_OTHER_LANGUAGE_PACKAGES = ActionNode( + key="Required Other language third-party packages", + expected_type=list[str], + instruction="List down the required packages for languages other than Python.", + example=["No third-party dependencies required"] +) + +LOGIC_ANALYSIS = ActionNode( + key="Logic Analysis", + expected_type=list[list[str]], + instruction="Provide a list of files with the classes/methods/functions to be implemented, " + "including dependency analysis and imports.", + example=[["game.py", "Contains Game class and ... functions"], + ["main.py", "Contains main function, from game import Game"]] +) + +TASK_LIST = ActionNode( + key="Task list", + expected_type=list[str], + instruction="Break down the tasks into a list of filenames, prioritized by dependency order.", + example=["game.py", "main.py"] +) + +FULL_API_SPEC = ActionNode( + key="Full API spec", + expected_type=str, + instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend.", + example="openapi: 3.0.0 ..." +) + +SHARED_KNOWLEDGE = ActionNode( + key="Shared Knowledge", + expected_type=str, + instruction="Detail any shared knowledge, like common utility functions or configuration variables.", + example="'game.py' contains functions shared across the project." +) + +ANYTHING_UNCLEAR_PM = ActionNode( + key="Anything UNCLEAR", + expected_type=str, + instruction="Mention any unclear aspects in the project management context and try to clarify them.", + example="Clarification needed on how to start and initialize third-party libraries." +) + +NODES = [ + REQUIRED_PYTHON_PACKAGES, + REQUIRED_OTHER_LANGUAGE_PACKAGES, + LOGIC_ANALYSIS, + TASK_LIST, + FULL_API_SPEC, + SHARED_KNOWLEDGE, + ANYTHING_UNCLEAR_PM +] + + +PM_NODE = ActionNode.from_children("PM_NODE", NODES) + + +def main(): + prompt = PM_NODE.compile(context="") + logger.info(prompt) + + +if __name__ == '__main__': + main() diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 0594d116e..e61743e7f 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -14,9 +14,11 @@ import json from pathlib import Path -from typing import List +# from typing import List from metagpt.actions import Action, ActionOutput +from metagpt.actions.action_node import ActionNode +from metagpt.actions.write_prd_an import WRITE_PRD_NODE, WP_ISSUE_TYPE_NODE, WP_IS_RELATIVE_NODE from metagpt.actions.fix_bug import FixBug from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG @@ -31,293 +33,26 @@ from metagpt.schema import Document, Documents, Message, BugFixContext from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository -from metagpt.utils.get_template import get_template +# from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file -templates = { - "json": { - "PROMPT_TEMPLATE": """ -# Context -{{ - "Original Requirements": "{requirements}", - "Search Information": "" -}} +CONTEXT_TEMPLATE = """ +### Project Name +{project_name} -## Format example -{format_example} ------ -Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. -ATTENTION: Output carefully referenced "Format example" in format. - -## YOU NEED TO FULFILL THE BELOW JSON DOC - -{{ - "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. - "Original Requirements": "", # str, place the polished complete original requirements here - "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. - "Search Information": "", - "Requirements": "", - "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. - "User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories - "Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses - # Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - "Competitive Quadrant Chart": "quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - Campaign A: [0.3, 0.6] - Campaign B: [0.45, 0.23] - Campaign C: [0.57, 0.69] - Campaign D: [0.78, 0.34] - Campaign E: [0.40, 0.34] - Campaign F: [0.35, 0.78]", - "Requirement Analysis": "", # Provide as Plain text. - "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards - "UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. - "Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it. -}} - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, -and only output the json inside this tag, nothing else -""", - "FORMAT_EXAMPLE": """ -[CONTENT] -{{ - "Language": "", - "Original Requirements": "", - "Project Name": "{project_name}", - "Search Information": "", - "Requirements": "", - "Product Goals": [], - "User Stories": [], - "Competitive Analysis": [], - "Competitive Quadrant Chart": "quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - Campaign A: [0.3, 0.6] - Campaign B: [0.45, 0.23] - Campaign C: [0.57, 0.69] - Campaign D: [0.78, 0.34] - Campaign E: [0.40, 0.34] - Campaign F: [0.35, 0.78]", - "Requirement Analysis": "", - "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], - "UI Design draft": "", - "Anything UNCLEAR": "", -}} -[/CONTENT] -""", - }, - "markdown": { - "PROMPT_TEMPLATE": """ -# Context -## Original Requirements +### Original Requirements {requirements} -## Search Information -{search_information} - -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` - -## Format example -{format_example} ------ -Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Language: Please use the same language as the user requirement to answer, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. - -## Language: Provide as Plain text, use the same language as the user requirement. - -## Original Requirements: Provide as Plain text, place the polished complete original requirements here - -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. -""", - "FORMAT_EXAMPLE": """ ---- -## Original Requirements -The user ... - -## Product Goals -```python -[ - "Create a ...", -] -``` - -## User Stories -```python -[ - "As a user, ...", -] -``` - -## Competitive Analysis -```python -[ - "Python Snake Game: ...", -] -``` - -## Competitive Quadrant Chart -```mermaid -quadrantChart - title Reach and engagement of campaigns - ... - "Our Target Product": [0.6, 0.7] -``` - -## Requirement Analysis -The product should be a ... - -## Requirement Pool -```python -[ - ["End game ...", "P0"] -] -``` - -## UI Design draft -Give a basic function description, and a draft - -## Anything UNCLEAR -There are no unclear points. ---- -""", - }, -} - -OUTPUT_MAPPING = { - "Language": (str, ...), - "Original Requirements": (str, ...), - "Project Name": (str, ...), - "Product Goals": (List[str], ...), - "User Stories": (List[str], ...), - "Competitive Analysis": (List[str], ...), - "Competitive Quadrant Chart": (str, ...), - "Requirement Analysis": (str, ...), - "Requirement Pool": (List[List[str]], ...), - "UI Design draft": (str, ...), - "Anything UNCLEAR": (str, ...), -} - -IS_RELATIVE_PROMPT = """ -## PRD: -{old_prd} - -## New Requirement: -{requirements} - -___ -You are a professional product manager; You need to assess whether the new requirements are relevant to the existing PRD to determine whether to merge the new requirements into this PRD. -Is the newly added requirement in "New Requirement" related to the PRD? -Respond with `YES` if it is related, `NO` if it is not, and provide the reasons. Return the response in JSON format. +### Search Information +- """ -MERGE_PROMPT = """ -# Context -## Original Requirements -{requirements} - - -## Old PRD +NEW_REQ_TEMPLATE = """ +### Legacy Content {old_prd} ------ -Role: You are a professional product manager; The goal is to incorporate the newly added requirements from the "Original Requirements" into the existing Product Requirements Document (PRD) in the "Old PRD" in order to design a concise, usable, and efficient product. -Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design -ATTENTION: Output carefully referenced "Old PRD" in format. -## YOU NEED TO FULFILL THE BELOW JSON DOC - -{{ - "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. - "Original Requirements": "", # str, place the polished complete original requirements here - "Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. - "Search Information": "", - "Requirements": "", - "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. - "User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories - "Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses - # Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - "Competitive Quadrant Chart": "quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - Campaign A: [0.3, 0.6] - Campaign B: [0.45, 0.23] - Campaign C: [0.57, 0.69] - Campaign D: [0.78, 0.34] - Campaign E: [0.40, 0.34] - Campaign F: [0.35, 0.78]", - "Requirement Analysis": "", # Provide as Plain text. - "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards - "UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. - "Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it. -}} - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old PRD" format, -and only output the json inside this tag, nothing else -""" - -IS_BUGFIX_PROMPT = """ -{content} - -___ -You are a professional product manager; You need to determine whether the above content describes a requirement or provides feedback about a bug. -Respond with `YES` if it is a feedback about a bug, `NO` if it is not, and provide the reasons. Return the response in JSON format like below: - -```json -{{ - "is_bugfix": ..., # `YES` or `NO` - "reason": ..., # reason string -}} -``` +### New Requirements +{requirements} """ @@ -335,7 +70,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") bug_fix = BugFixContext(filename=BUGFIX_FILENAME) return Message(content=bug_fix.json(), instruct_content=bug_fix, - role=self.profile, + role="", cause_by=FixBug, sent_from=self, send_to="Alex", # the name of Engineer @@ -353,7 +88,7 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc - logger.info(f"REWRITE PRD:{prd_doc.filename}") + logger.info(f"rewrite prd: {prd_doc.filename}") # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( @@ -367,47 +102,32 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) # optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) - async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: - sas = SearchAndSummarize() - # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) - rsp = "" - info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" - if sas.result: - logger.info(sas.result) - logger.info(rsp) - - # logger.info(format) - prompt_template, format_example = get_template(templates, format) + async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format) -> ActionOutput: + # sas = SearchAndSummarize() + # # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) + # rsp = "" + # info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" + # if sas.result: + # logger.info(sas.result) + # logger.info(rsp) project_name = CONFIG.project_name if CONFIG.project_name else "" - format_example = format_example.format(project_name=project_name) - # logger.info(prompt_template) - # logger.info(format_example) - prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example, project_name=project_name - ) - # logger.info(prompt) - # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) - prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) - await self._rename_workspace(prd) - return prd + context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name) + node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, to=format) + await self._rename_workspace(node) + return node - async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: - prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) - res = await self._aask(prompt=prompt) - logger.info(f"REQ-RELATIVE: [{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") - if "YES" in res: - return True - return False + async def _is_relative(self, new_requirement_doc, old_prd_doc) -> bool: + context = NEW_REQ_TEMPLATE.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) + node = await WP_IS_RELATIVE_NODE.fill(context, self.llm) + return node.get("is_relative") == "YES" async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: if not CONFIG.project_name: CONFIG.project_name = Path(CONFIG.project_path).name - prompt = MERGE_PROMPT.format( - requirements=new_requirement_doc.content, old_prd=prd_doc.content, project_name=CONFIG.project_name - ) - prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) - prd_doc.content = prd.instruct_content.json(ensure_ascii=False) - await self._rename_workspace(prd) + prompt = NEW_REQ_TEMPLATE.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, to=format) + prd_doc.content = node.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(node) return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: @@ -418,7 +138,7 @@ async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **k filename=FileRepository.new_filename() + ".json", content=prd.instruct_content.json(ensure_ascii=False), ) - elif await self._is_relative_to(requirement_doc, prd_doc): + elif await self._is_relative(requirement_doc, prd_doc): new_prd_doc = await self._merge(requirement_doc, prd_doc) else: return None @@ -453,17 +173,13 @@ async def _rename_workspace(prd): return if not CONFIG.project_name: - if isinstance(prd, ActionOutput): + if isinstance(prd, ActionOutput) or isinstance(prd, ActionNode): ws_name = prd.instruct_content.dict()["Project Name"] else: ws_name = CodeParser.parse_str(block="Project Name", text=prd) CONFIG.project_name = ws_name CONFIG.git_repo.rename_root(CONFIG.project_name) - async def _is_bugfix(self, content): - prompt = IS_BUGFIX_PROMPT.format(content=content) - res = await self._aask(prompt=prompt) - logger.info(f"IS_BUGFIX:{res}") - if "YES" in res: - return True - return False + async def _is_bugfix(self, context) -> bool: + node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm) + return node.get("issue_type") == "BUG" diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py new file mode 100644 index 000000000..7368621ea --- /dev/null +++ b/metagpt/actions/write_prd_an.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/14 11:40 +@Author : alexanderwu +@File : write_prd_an.py +""" + +from metagpt.actions.action_node import ActionNode +from metagpt.logs import logger + +LANGUAGE = ActionNode( + key="Language", + expected_type=str, + instruction="Provide the language used in the project, typically matching the user's requirement language.", + example="en_us" +) + +ORIGINAL_REQUIREMENTS = ActionNode( + key="Original Requirements", + expected_type=str, + instruction="Place the polished, complete original requirements here.", + example="The game should have a leaderboard and multiple difficulty levels." +) + +PROJECT_NAME = ActionNode( + key="Project Name", + expected_type=str, + instruction="Name the project using snake case style, like 'game_2048' or 'simple_crm'.", + example="game_2048" +) + +PRODUCT_GOALS = ActionNode( + key="Product Goals", + expected_type=list[str], + instruction="Provide up to three clear, orthogonal product goals.", + example=["Create an engaging user experience", + "Ensure high performance", + "Provide customizable features"] +) + +USER_STORIES = ActionNode( + key="User Stories", + expected_type=list[str], + instruction="Provide up to five scenario-based user stories.", + example=["As a user, I want to be able to choose difficulty levels", + "As a player, I want to see my score after each game"] +) + +COMPETITIVE_ANALYSIS = ActionNode( + key="Competitive Analysis", + expected_type=list[str], + instruction="Provide analyses for up to seven competitive products.", + example=["Python Snake Game: Simple interface, lacks advanced features"] +) + +COMPETITIVE_QUADRANT_CHART = ActionNode( + key="Competitive Quadrant Chart", + expected_type=str, + instruction="Use mermaid quadrantChart syntax. Distribute scores evenly between 0 and 1", + example="""quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + "Campaign: A": [0.3, 0.6] + "Campaign B": [0.45, 0.23] + "Campaign C": [0.57, 0.69] + "Campaign D": [0.78, 0.34] + "Campaign E": [0.40, 0.34] + "Campaign F": [0.35, 0.78] + "Our Target Product": [0.5, 0.6]""" +) + +REQUIREMENT_ANALYSIS = ActionNode( + key="Requirement Analysis", + expected_type=str, + instruction="Provide a detailed analysis of the requirements.", + example="The product should be user-friendly and performance-optimized." +) + +REQUIREMENT_POOL = ActionNode( + key="Requirement Pool", + expected_type=list[list[str]], + instruction="List down the requirements with their priority (P0, P1, P2).", + example=[["P0", "High priority requirement"], ["P1", "Medium priority requirement"]] +) + +UI_DESIGN_DRAFT = ActionNode( + key="UI Design draft", + expected_type=str, + instruction="Provide a simple description of UI elements, functions, style, and layout.", + example="Basic function description with a simple style and layout." +) + +ANYTHING_UNCLEAR = ActionNode( + key="Anything UNCLEAR", + expected_type=str, + instruction="Mention any aspects of the project that are unclear and try to clarify them.", + example="..." +) + +ISSUE_TYPE = ActionNode( + key="issue_type", + expected_type=str, + instruction="Answer BUG/REQUIREMENT. If it is a bugfix, answer Bug, otherwise answer Requirement", + example="BUG" +) + +IS_RELATIVE = ActionNode( + key="is_relative", + expected_type=str, + instruction="Answer YES/NO. If the requirement is related to the old PRD, answer YES, otherwise NO", + example="YES" +) + +REASON = ActionNode( + key="reason", + expected_type=str, + instruction="Explain the reasoning process from question to answer", + example="..." +) + + +NODES = [ + LANGUAGE, + ORIGINAL_REQUIREMENTS, + PROJECT_NAME, + PRODUCT_GOALS, + USER_STORIES, + COMPETITIVE_ANALYSIS, + COMPETITIVE_QUADRANT_CHART, + REQUIREMENT_ANALYSIS, + REQUIREMENT_POOL, + UI_DESIGN_DRAFT, + ANYTHING_UNCLEAR +] + +WRITE_PRD_NODE = ActionNode.from_children("WritePRD", NODES) +WP_ISSUE_TYPE_NODE = ActionNode.from_children("WP_ISSUE_TYPE", [ISSUE_TYPE, REASON]) +WP_IS_RELATIVE_NODE = ActionNode.from_children("WP_IS_RELATIVE", [IS_RELATIVE, REASON]) + + +def main(): + prompt = WRITE_PRD_NODE.compile(context="") + logger.info(prompt) + + +if __name__ == '__main__': + main() diff --git a/metagpt/environment.py b/metagpt/environment.py index 7d1e307f3..89b6f9d46 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -82,7 +82,7 @@ async def run(self, k=1): futures.append(future) await asyncio.gather(*futures) - logger.info(f"is idle: {self.is_idle}") + logger.debug(f"is idle: {self.is_idle}") def get_roles(self) -> dict[str, Role]: """获得环境内的所有角色 diff --git a/metagpt/llm.py b/metagpt/llm.py index d8d06c0a1..eaa4880a5 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -12,11 +12,12 @@ from metagpt.provider.openai_api import OpenAIGPTAPI from metagpt.provider.spark_api import SparkAPI from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +from metagpt.provider.base_gpt_api import BaseGPTAPI _ = HumanProvider() # Avoid pre-commit error -def LLM() -> "BaseGPTAPI": +def LLM() -> BaseGPTAPI: """initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further if CONFIG.openai_api_key: diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 15d5fe5b1..b80ef85be 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -26,8 +26,8 @@ def __init__( self, name: str = "Bob", profile: str = "Architect", - goal: str = "Design a concise, usable, complete python system", - constraints: str = "Try to specify good open source tools as much as possible", + goal: str = "design a concise, usable, complete software system", + constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries" ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index cedd2101f..844f3589d 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -71,14 +71,15 @@ def __init__( self, name: str = "Alex", profile: str = "Engineer", - goal: str = "Write elegant, readable, extensible, efficient code", - constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", + goal: str = "write elegant, readable, extensible, efficient code", + constraints: str = "the code should conform to standards like PEP8 and be modular and maintainable", n_borg: int = 1, use_code_review: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review + self._init_actions([WriteCode]) self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug]) self.code_todos = [] self.summarize_todos = [] @@ -198,11 +199,11 @@ async def _think(self) -> Action | None: return None msg = self._rc.news[0] if msg.cause_by in write_code_filters: - logger.info(f"TODO WriteCode:{msg.json()}") + logger.debug(f"TODO WriteCode:{msg.json()}") await self._new_code_actions(bug_fix=msg.cause_by == any_to_str(FixBug)) return self._rc.todo if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): - logger.info(f"TODO SummarizeCode:{msg.json()}") + logger.debug(f"TODO SummarizeCode:{msg.json()}") await self._new_summarize_actions() return self._rc.todo return None diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 7e7c5699d..37090b24f 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -25,7 +25,8 @@ def __init__( self, name: str = "Eve", profile: str = "Project Manager", - goal: str = "Improve team efficiency and deliver with quality and quantity", + goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task " + "dependencies to start with the prerequisite modules", constraints: str = "", ) -> None: """ diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 52ac3cf28..7c9341adb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -26,6 +26,7 @@ from pydantic import BaseModel, Field from metagpt.actions import Action, ActionOutput +from metagpt.actions.action_node import ActionNode from metagpt.config import CONFIG from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger @@ -156,7 +157,7 @@ def _init_actions(self, actions): f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances" ) i = action - i.set_env(self._rc.env) + # i.set_env(self._rc.env) i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) self._states.append(f"{idx}. {action}") @@ -278,7 +279,7 @@ async def _think(self) -> None: async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.important_memory) - if isinstance(response, ActionOutput): + if isinstance(response, ActionOutput) or isinstance(response, ActionNode): msg = Message( content=response.content, instruct_content=response.instruct_content, diff --git a/metagpt/roles/searcher.py b/metagpt/roles/searcher.py index bee8d3986..5760202ff 100644 --- a/metagpt/roles/searcher.py +++ b/metagpt/roles/searcher.py @@ -8,6 +8,7 @@ the `cause_by` value in the `Message` to a string to support the new message distribution feature. """ from metagpt.actions import ActionOutput, SearchAndSummarize +from metagpt.actions.action_node import ActionNode from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -58,7 +59,7 @@ async def _act_sp(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.memory.get(k=0)) - if isinstance(response, ActionOutput): + if isinstance(response, ActionOutput) or isinstance(response, ActionNode): msg = Message( content=response.content, instruct_content=response.instruct_content, From 39cb66359505edef07b9e1fb5f5c1f341372bcec Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 14 Dec 2023 16:21:56 +0800 Subject: [PATCH 216/232] fix typo --- metagpt/actions/action_node.py | 2 +- metagpt/actions/write_prd_an.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 178986ebe..96c175ccb 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -285,7 +285,7 @@ async def fill(self, context, llm, to="json"): def action_node_from_tuple_example(): # 示例：列表中包含元组 list_of_tuples = [ - ("key1", int, "Instruction 1", "Example 1") + ("key1", str, "Instruction 1", "Example 1") ] # 从列表中创建 ActionNode 实例 diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index 7368621ea..0781760ba 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -106,7 +106,7 @@ ISSUE_TYPE = ActionNode( key="issue_type", expected_type=str, - instruction="Answer BUG/REQUIREMENT. If it is a bugfix, answer Bug, otherwise answer Requirement", + instruction="Answer BUG/REQUIREMENT. If it is a bugfix, answer BUG, otherwise answer Requirement", example="BUG" ) From 609d75a07eba441dcba4c3c2ea0644f9836f6d5a Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 14 Dec 2023 18:06:43 +0800 Subject: [PATCH 217/232] add programming language as input, add complex strgy to ActionNode.fill method, fix quadrantChart in chinese etc. --- metagpt/actions/action_node.py | 74 ++++++++++++++++++++---- metagpt/actions/project_management_an.py | 2 +- metagpt/actions/write_prd_an.py | 24 +++++--- 3 files changed, 80 insertions(+), 20 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 96c175ccb..b1fbdaae9 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -28,6 +28,8 @@ ## context {context} +----- + ## format example {example} @@ -38,7 +40,7 @@ {constraint} ## action -Fill in the above nodes based on the context. Answer in format example. +Fill in the above nodes based on the format example. """ @@ -108,6 +110,16 @@ def get_children_mapping(self) -> Dict[str, Type]: """获得子ActionNode的字典，以key索引""" return {k: (v.expected_type, ...) for k, v in self.children.items()} + def get_self_mapping(self) -> Dict[str, Type]: + """get self key: type mapping""" + return {self.key: (self.expected_type, ...)} + + def get_mapping(self, mode="children") -> Dict[str, Type]: + """get key: type mapping under mode""" + if mode == "children" or (mode=="auto" and self.children): + return self.get_children_mapping() + return self.get_self_mapping() + @classmethod def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): """基于pydantic v1的模型动态生成，用来检验结果类型正确性""" @@ -160,8 +172,8 @@ def create_children_class(self): mapping = self.get_children_mapping() return self.create_model_class(class_name, mapping) - def to_dict(self, format_func=None, mode="all") -> Dict: - """将当前节点与子节点都按照node: format的格式组织称字典""" + def to_dict(self, format_func=None, mode="auto") -> Dict: + """将当前节点与子节点都按照node: format的格式组织成字典""" # 如果没有提供格式化函数，使用默认的格式化方式 if format_func is None: @@ -171,7 +183,7 @@ def to_dict(self, format_func=None, mode="all") -> Dict: formatted_value = format_func(self) # 创建当前节点的键值对 - if mode == "children": + if mode == "children" or (mode == "auto" and self.children): node_dict = {} else: node_dict = {self.key: formatted_value} @@ -227,7 +239,7 @@ def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) mode="root": NotImplemented """ - # FIXME: json instruction会带来 "Project name": "web_2048 # 项目名称使用下划线", + # FIXME: json instruction会带来格式问题，如："Project name": "web_2048 # 项目名称使用下划线", self.instruction = self.compile_instruction(to="markdown", mode=mode) self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) prompt = template.format(context=context, example=self.example, instruction=self.instruction, @@ -268,19 +280,59 @@ async def _aask_v1( def get(self, key): return self.instruct_content.dict()[key] - async def fill(self, context, llm, to="json"): - """运行这个ActionNode，并且填槽，可以采用不同策略，比如只运行子节点""" - self.llm = llm - prompt = self.compile(context=context, to=to) - mapping = self.get_children_mapping() + def set_recursive(self, name, value): + setattr(self, name, value) + for _, i in self.children.items(): + i.set_recursive(name, value) + + def set_llm(self, llm): + self.set_recursive("llm", llm) + + def set_context(self, context): + self.set_recursive("context", context) + + async def simple_fill(self, to, mode): + prompt = self.compile(context=self.context, to=to, mode=mode) + mapping = self.get_mapping(mode) class_name = f"{self.key}_AN" - # 需要传入llm，并且实际在ActionNode中执行。需要规划好具体的执行方法 output = await self._aask_v1(prompt, class_name, mapping, format=to) self.content = output.content self.instruct_content = output.instruct_content return self + async def fill(self, context, llm, to="json", mode="auto", strgy="simple"): + """ Fill the node(s) with mode. + + :param context: Everything we should know when filling node. + :param llm: Large Language Model with pre-defined system message. + :param to: json/markdown, determine example and output format. + - json: it's easy to open source LLM with json format + - markdown: when generating code, markdown is always better + :param mode: auto/children/root + - auto: automated fill children's nodes and gather outputs, if no children, fill itself + - children: fill children's nodes and gather outputs + - root: fill root's node and gather output + :param strgy: simple/complex + - simple: run only once + - complex: run each node + :return: self + """ + self.set_llm(llm) + self.set_context(context) + + if strgy == "simple": + return await self.simple_fill(to, mode) + elif strgy == "complex": + # 这里隐式假设了拥有children + tmp = {} + for _, i in self.children.items(): + child = await i.simple_fill(to, mode) + tmp.update(child.instruct_content.dict()) + cls = self.create_children_class() + self.instruct_content = cls(**tmp) + return self + def action_node_from_tuple_example(): # 示例：列表中包含元组 diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py index aa7cdcde2..9849cb7b3 100644 --- a/metagpt/actions/project_management_an.py +++ b/metagpt/actions/project_management_an.py @@ -28,7 +28,7 @@ instruction="Provide a list of files with the classes/methods/functions to be implemented, " "including dependency analysis and imports.", example=[["game.py", "Contains Game class and ... functions"], - ["main.py", "Contains main function, from game import Game"]] + ["main.py", "Contains main function, depends on game.py"]] ) TASK_LIST = ActionNode( diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index 0781760ba..cbcf920b9 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -16,6 +16,13 @@ example="en_us" ) +PROGRAMMING_LANGUAGE = ActionNode( + key="Programming Language", + expected_type=str, + instruction="Python/JavaScript or other mainstream programming language.", + example="Python" +) + ORIGINAL_REQUIREMENTS = ActionNode( key="Original Requirements", expected_type=str, @@ -59,14 +66,14 @@ expected_type=str, instruction="Use mermaid quadrantChart syntax. Distribute scores evenly between 0 and 1", example="""quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] + title "Reach and engagement of campaigns" + x-axis "Low Reach" --> "High Reach" + y-axis "Low Engagement" --> "High Engagement" + quadrant-1 "We should expand" + quadrant-2 "Need to promote" + quadrant-3 "Re-evaluate" + quadrant-4 "May be improved" + "Campaign A": [0.3, 0.6] "Campaign B": [0.45, 0.23] "Campaign C": [0.57, 0.69] "Campaign D": [0.78, 0.34] @@ -127,6 +134,7 @@ NODES = [ LANGUAGE, + PROGRAMMING_LANGUAGE, ORIGINAL_REQUIREMENTS, PROJECT_NAME, PRODUCT_GOALS, From 290fb8b8d053a4d1441ac64fff60550f0b9e18e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 14 Dec 2023 20:44:27 +0800 Subject: [PATCH 218/232] refactor: format --- .gitignore | 1 + metagpt/actions/action_node.py | 31 ++- metagpt/actions/design_api_an.py | 27 +-- metagpt/actions/project_management.py | 7 +- metagpt/actions/project_management_an.py | 24 +- metagpt/actions/write_prd_an.py | 41 ++-- metagpt/provider/fireworks_api.py | 3 +- metagpt/provider/open_llm_api.py | 7 +- .../postprecess/base_postprecess_plugin.py | 22 +- .../postprecess/llm_output_postprecess.py | 11 +- metagpt/roles/architect.py | 2 +- metagpt/roles/project_manager.py | 2 +- metagpt/roles/qa_engineer.py | 4 +- metagpt/schema.py | 16 +- metagpt/utils/ahttp_client.py | 56 ++--- metagpt/utils/git_repository.py | 9 +- metagpt/utils/repair_llm_raw_output.py | 29 ++- metagpt/utils/utils.py | 11 +- tests/metagpt/test_llm.py | 1 + tests/metagpt/utils/test_ahttp_client.py | 17 +- .../utils/test_repair_llm_raw_output.py | 225 +++++++++--------- 21 files changed, 262 insertions(+), 284 deletions(-) diff --git a/.gitignore b/.gitignore index e03eab3d3..0ac318ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ cover/ # Django stuff: *.log +logs local_settings.py db.sqlite3 db.sqlite3-journal diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 96c175ccb..ae40913e0 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -5,13 +5,12 @@ @Author : alexanderwu @File : action_node.py """ -import re -from typing import Dict, Type, List, Any, Tuple, Optional import json +import re +from typing import Any, Dict, List, Optional, Type from pydantic import BaseModel, create_model, root_validator, validator -# , model_validator, field_validator -from tenacity import wait_random_exponential, stop_after_attempt, retry +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions import ActionOutput from metagpt.llm import BaseGPTAPI @@ -51,6 +50,7 @@ def dict_to_markdown(d, prefix="-", postfix="\n"): class ActionNode: """ActionNode is a tree of nodes.""" + # Action Strgy # - sop: 仅使用一级SOP # - complex: 使用一级SOP+自定义策略填槽 @@ -72,8 +72,7 @@ class ActionNode: content: str instruct_content: BaseModel - def __init__(self, key, expected_type, instruction, example, content="", - children=None): + def __init__(self, key, expected_type, instruction, example, content="", children=None): self.key = key self.expected_type = expected_type self.instruction = instruction @@ -82,8 +81,9 @@ def __init__(self, key, expected_type, instruction, example, content="", self.children = children if children is not None else {} def __str__(self): - return f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" \ - f", {self.content}, {self.children}" + return ( + f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" f", {self.content}, {self.children}" + ) def __repr__(self): return self.__str__() @@ -136,7 +136,7 @@ def create_model_class_v2(cls, class_name: str, mapping: Dict[str, Type]): """基于pydantic v2的模型动态生成，用来检验结果类型正确性，待验证""" new_class = create_model(class_name, **mapping) - @model_validator(mode='before') + @model_validator(mode="before") def check_missing_fields(data): required_fields = set(mapping.keys()) missing_fields = required_fields - set(data.keys()) @@ -144,7 +144,7 @@ def check_missing_fields(data): raise ValueError(f"Missing fields: {missing_fields}") return data - @field_validator('*') + @field_validator("*") def check_name(v: Any, field: str) -> Any: if field not in mapping.keys(): raise ValueError(f"Unrecognized block: {field}") @@ -230,8 +230,9 @@ def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) # FIXME: json instruction会带来 "Project name": "web_2048 # 项目名称使用下划线", self.instruction = self.compile_instruction(to="markdown", mode=mode) self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) - prompt = template.format(context=context, example=self.example, instruction=self.instruction, - constraint=CONSTRAINT) + prompt = template.format( + context=context, example=self.example, instruction=self.instruction, constraint=CONSTRAINT + ) return prompt @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) @@ -284,9 +285,7 @@ async def fill(self, context, llm, to="json"): def action_node_from_tuple_example(): # 示例：列表中包含元组 - list_of_tuples = [ - ("key1", str, "Instruction 1", "Example 1") - ] + list_of_tuples = [("key1", str, "Instruction 1", "Example 1")] # 从列表中创建 ActionNode 实例 nodes = [ActionNode(*data) for data in list_of_tuples] @@ -294,5 +293,5 @@ def action_node_from_tuple_example(): logger.info(i) -if __name__ == '__main__': +if __name__ == "__main__": action_node_from_tuple_example() diff --git a/metagpt/actions/design_api_an.py b/metagpt/actions/design_api_an.py index 2db203606..0a303cdd5 100644 --- a/metagpt/actions/design_api_an.py +++ b/metagpt/actions/design_api_an.py @@ -6,52 +6,49 @@ @File : design_api_an.py """ from metagpt.actions.action_node import ActionNode -from metagpt.utils.mermaid import MMC1, MMC2 from metagpt.logs import logger +from metagpt.utils.mermaid import MMC1, MMC2 IMPLEMENTATION_APPROACH = ActionNode( key="Implementation approach", expected_type=str, instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework", - example="We will ..." + example="We will ...", ) PROJECT_NAME = ActionNode( - key="Project name", - expected_type=str, - instruction="The project name with underline", - example="game_2048" + key="Project name", expected_type=str, instruction="The project name with underline", example="game_2048" ) FILE_LIST = ActionNode( key="File list", expected_type=list[str], instruction="Only need relative paths. ALWAYS write a main.py or app.py here", - example=['main.py', 'game.py'] + example=["main.py", "game.py"], ) DATA_STRUCTURES_AND_INTERFACES = ActionNode( key="Data structures and interfaces", expected_type=str, instruction="Use mermaid classDiagram code syntax, including classes, method(__init__ etc.) and functions with type" - " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " - "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", - example=MMC1 + " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " + "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", + example=MMC1, ) PROGRAM_CALL_FLOW = ActionNode( key="Program call flow", expected_type=str, instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE " - "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", - example=MMC2 + "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", + example=MMC2, ) ANYTHING_UNCLEAR = ActionNode( key="Anything UNCLEAR", expected_type=str, instruction="Mention unclear project aspects, then try to clarify it.", - example="Clarification needed on third-party API integration, ..." + example="Clarification needed on third-party API integration, ...", ) NODES = [ @@ -60,7 +57,7 @@ FILE_LIST, DATA_STRUCTURES_AND_INTERFACES, PROGRAM_CALL_FLOW, - ANYTHING_UNCLEAR + ANYTHING_UNCLEAR, ] DESIGN_API_NODE = ActionNode.from_children("DesignAPI", NODES) @@ -71,5 +68,5 @@ def main(): logger.info(prompt) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 29e3bed3e..c95be4012 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -10,7 +10,6 @@ 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ import json -# from typing import List from metagpt.actions import ActionOutput from metagpt.actions.action import Action @@ -25,6 +24,9 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository + +# from typing import List + # from metagpt.utils.get_template import get_template NEW_REQ_TEMPLATE = """ @@ -97,7 +99,8 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format): async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content) node = await PM_NODE.fill(context, self.llm, format) - return node + task_doc.content = node.content + return task_doc @staticmethod async def _update_requirements(doc): diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py index aa7cdcde2..e03af36d7 100644 --- a/metagpt/actions/project_management_an.py +++ b/metagpt/actions/project_management_an.py @@ -12,51 +12,53 @@ key="Required Python packages", expected_type=list[str], instruction="Provide required Python packages in requirements.txt format.", - example=["flask==1.1.2", "bcrypt==3.2.0"] + example=["flask==1.1.2", "bcrypt==3.2.0"], ) REQUIRED_OTHER_LANGUAGE_PACKAGES = ActionNode( key="Required Other language third-party packages", expected_type=list[str], instruction="List down the required packages for languages other than Python.", - example=["No third-party dependencies required"] + example=["No third-party dependencies required"], ) LOGIC_ANALYSIS = ActionNode( key="Logic Analysis", expected_type=list[list[str]], instruction="Provide a list of files with the classes/methods/functions to be implemented, " - "including dependency analysis and imports.", - example=[["game.py", "Contains Game class and ... functions"], - ["main.py", "Contains main function, from game import Game"]] + "including dependency analysis and imports.", + example=[ + ["game.py", "Contains Game class and ... functions"], + ["main.py", "Contains main function, from game import Game"], + ], ) TASK_LIST = ActionNode( key="Task list", expected_type=list[str], instruction="Break down the tasks into a list of filenames, prioritized by dependency order.", - example=["game.py", "main.py"] + example=["game.py", "main.py"], ) FULL_API_SPEC = ActionNode( key="Full API spec", expected_type=str, instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend.", - example="openapi: 3.0.0 ..." + example="openapi: 3.0.0 ...", ) SHARED_KNOWLEDGE = ActionNode( key="Shared Knowledge", expected_type=str, instruction="Detail any shared knowledge, like common utility functions or configuration variables.", - example="'game.py' contains functions shared across the project." + example="'game.py' contains functions shared across the project.", ) ANYTHING_UNCLEAR_PM = ActionNode( key="Anything UNCLEAR", expected_type=str, instruction="Mention any unclear aspects in the project management context and try to clarify them.", - example="Clarification needed on how to start and initialize third-party libraries." + example="Clarification needed on how to start and initialize third-party libraries.", ) NODES = [ @@ -66,7 +68,7 @@ TASK_LIST, FULL_API_SPEC, SHARED_KNOWLEDGE, - ANYTHING_UNCLEAR_PM + ANYTHING_UNCLEAR_PM, ] @@ -78,5 +80,5 @@ def main(): logger.info(prompt) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index 0781760ba..849150f6c 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -13,45 +13,45 @@ key="Language", expected_type=str, instruction="Provide the language used in the project, typically matching the user's requirement language.", - example="en_us" + example="en_us", ) ORIGINAL_REQUIREMENTS = ActionNode( key="Original Requirements", expected_type=str, instruction="Place the polished, complete original requirements here.", - example="The game should have a leaderboard and multiple difficulty levels." + example="The game should have a leaderboard and multiple difficulty levels.", ) PROJECT_NAME = ActionNode( key="Project Name", expected_type=str, instruction="Name the project using snake case style, like 'game_2048' or 'simple_crm'.", - example="game_2048" + example="game_2048", ) PRODUCT_GOALS = ActionNode( key="Product Goals", expected_type=list[str], instruction="Provide up to three clear, orthogonal product goals.", - example=["Create an engaging user experience", - "Ensure high performance", - "Provide customizable features"] + example=["Create an engaging user experience", "Ensure high performance", "Provide customizable features"], ) USER_STORIES = ActionNode( key="User Stories", expected_type=list[str], instruction="Provide up to five scenario-based user stories.", - example=["As a user, I want to be able to choose difficulty levels", - "As a player, I want to see my score after each game"] + example=[ + "As a user, I want to be able to choose difficulty levels", + "As a player, I want to see my score after each game", + ], ) COMPETITIVE_ANALYSIS = ActionNode( key="Competitive Analysis", expected_type=list[str], instruction="Provide analyses for up to seven competitive products.", - example=["Python Snake Game: Simple interface, lacks advanced features"] + example=["Python Snake Game: Simple interface, lacks advanced features"], ) COMPETITIVE_QUADRANT_CHART = ActionNode( @@ -72,56 +72,53 @@ "Campaign D": [0.78, 0.34] "Campaign E": [0.40, 0.34] "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6]""" + "Our Target Product": [0.5, 0.6]""", ) REQUIREMENT_ANALYSIS = ActionNode( key="Requirement Analysis", expected_type=str, instruction="Provide a detailed analysis of the requirements.", - example="The product should be user-friendly and performance-optimized." + example="The product should be user-friendly and performance-optimized.", ) REQUIREMENT_POOL = ActionNode( key="Requirement Pool", expected_type=list[list[str]], instruction="List down the requirements with their priority (P0, P1, P2).", - example=[["P0", "High priority requirement"], ["P1", "Medium priority requirement"]] + example=[["P0", "High priority requirement"], ["P1", "Medium priority requirement"]], ) UI_DESIGN_DRAFT = ActionNode( key="UI Design draft", expected_type=str, instruction="Provide a simple description of UI elements, functions, style, and layout.", - example="Basic function description with a simple style and layout." + example="Basic function description with a simple style and layout.", ) ANYTHING_UNCLEAR = ActionNode( key="Anything UNCLEAR", expected_type=str, instruction="Mention any aspects of the project that are unclear and try to clarify them.", - example="..." + example="...", ) ISSUE_TYPE = ActionNode( key="issue_type", expected_type=str, instruction="Answer BUG/REQUIREMENT. If it is a bugfix, answer BUG, otherwise answer Requirement", - example="BUG" + example="BUG", ) IS_RELATIVE = ActionNode( key="is_relative", expected_type=str, instruction="Answer YES/NO. If the requirement is related to the old PRD, answer YES, otherwise NO", - example="YES" + example="YES", ) REASON = ActionNode( - key="reason", - expected_type=str, - instruction="Explain the reasoning process from question to answer", - example="..." + key="reason", expected_type=str, instruction="Explain the reasoning process from question to answer", example="..." ) @@ -136,7 +133,7 @@ REQUIREMENT_ANALYSIS, REQUIREMENT_POOL, UI_DESIGN_DRAFT, - ANYTHING_UNCLEAR + ANYTHING_UNCLEAR, ] WRITE_PRD_NODE = ActionNode.from_children("WritePRD", NODES) @@ -149,5 +146,5 @@ def main(): logger.info(prompt) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/metagpt/provider/fireworks_api.py b/metagpt/provider/fireworks_api.py index 23126af2d..47ac9cf61 100644 --- a/metagpt/provider/fireworks_api.py +++ b/metagpt/provider/fireworks_api.py @@ -5,11 +5,10 @@ import openai from metagpt.config import CONFIG -from metagpt.provider.openai_api import OpenAIGPTAPI, CostManager, RateLimiter +from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter class FireWorksGPTAPI(OpenAIGPTAPI): - def __init__(self): self.__init_fireworks(CONFIG) self.llm = openai diff --git a/metagpt/provider/open_llm_api.py b/metagpt/provider/open_llm_api.py index a6820b42b..f421e30c8 100644 --- a/metagpt/provider/open_llm_api.py +++ b/metagpt/provider/open_llm_api.py @@ -4,13 +4,13 @@ import openai -from metagpt.logs import logger from metagpt.config import CONFIG -from metagpt.provider.openai_api import OpenAIGPTAPI, CostManager, RateLimiter +from metagpt.logs import logger +from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter class OpenLLMCostManager(CostManager): - """ open llm model is self-host, it's free and without cost""" + """open llm model is self-host, it's free and without cost""" def update_cost(self, prompt_tokens, completion_tokens, model): """ @@ -32,7 +32,6 @@ def update_cost(self, prompt_tokens, completion_tokens, model): class OpenLLMGPTAPI(OpenAIGPTAPI): - def __init__(self): self.__init_openllm(CONFIG) self.llm = openai diff --git a/metagpt/provider/postprecess/base_postprecess_plugin.py b/metagpt/provider/postprecess/base_postprecess_plugin.py index 702a03194..0d1cfbb11 100644 --- a/metagpt/provider/postprecess/base_postprecess_plugin.py +++ b/metagpt/provider/postprecess/base_postprecess_plugin.py @@ -5,13 +5,15 @@ from typing import Union from metagpt.logs import logger -from metagpt.utils.repair_llm_raw_output import RepairType -from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, extract_content_from_output, \ - retry_parse_json_text +from metagpt.utils.repair_llm_raw_output import ( + RepairType, + extract_content_from_output, + repair_llm_raw_output, + retry_parse_json_text, +) class BasePostPrecessPlugin(object): - model = None # the plugin of the `model`, use to judge in `llm_postprecess` def run_repair_llm_output(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]: @@ -33,15 +35,15 @@ def run_repair_llm_output(self, output: str, schema: dict, req_key: str = "[/CON return parsed_data def run_repair_llm_raw_output(self, content: str, req_keys: list[str], repair_type: str = None) -> str: - """ inherited class can re-implement the function""" + """inherited class can re-implement the function""" return repair_llm_raw_output(content, req_keys=req_keys, repair_type=repair_type) def run_extract_content_from_output(self, content: str, right_key: str) -> str: - """ inherited class can re-implement the function""" + """inherited class can re-implement the function""" return extract_content_from_output(content, right_key=right_key) def run_retry_parse_json_text(self, content: str) -> Union[dict, list]: - """ inherited class can re-implement the function""" + """inherited class can re-implement the function""" logger.info(f"extracted json CONTENT from output:\n{content}") parsed_data = retry_parse_json_text(output=content) # should use output=content return parsed_data @@ -64,9 +66,5 @@ def run(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[d assert "/" in req_key # current, postprocess only deal the repair_llm_raw_output - new_output = self.run_repair_llm_output( - output=output, - schema=schema, - req_key=req_key - ) + new_output = self.run_repair_llm_output(output=output, schema=schema, req_key=req_key) return new_output diff --git a/metagpt/provider/postprecess/llm_output_postprecess.py b/metagpt/provider/postprecess/llm_output_postprecess.py index 4b5955061..85405543d 100644 --- a/metagpt/provider/postprecess/llm_output_postprecess.py +++ b/metagpt/provider/postprecess/llm_output_postprecess.py @@ -7,17 +7,14 @@ from metagpt.provider.postprecess.base_postprecess_plugin import BasePostPrecessPlugin -def llm_output_postprecess(output: str, schema: dict, req_key: str = "[/CONTENT]", - model_name: str = None) -> Union[dict, str]: +def llm_output_postprecess( + output: str, schema: dict, req_key: str = "[/CONTENT]", model_name: str = None +) -> Union[dict, str]: """ default use BasePostPrecessPlugin if there is not matched plugin. """ # TODO choose different model's plugin according to the model_name postprecess_plugin = BasePostPrecessPlugin() - result = postprecess_plugin.run( - output=output, - schema=schema, - req_key=req_key - ) + result = postprecess_plugin.run(output=output, schema=schema, req_key=req_key) return result diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index b80ef85be..2c0bdd1d6 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -27,7 +27,7 @@ def __init__( name: str = "Bob", profile: str = "Architect", goal: str = "design a concise, usable, complete software system", - constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries" + constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries", ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 37090b24f..bfe1be251 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -26,7 +26,7 @@ def __init__( name: str = "Eve", profile: str = "Project Manager", goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task " - "dependencies to start with the prerequisite modules", + "dependencies to start with the prerequisite modules", constraints: str = "", ) -> None: """ diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 15a01b9e9..c1573e63b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -14,9 +14,7 @@ @Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results of SummarizeCode. """ -from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest - -# from metagpt.const import WORKSPACE_ROOT +from metagpt.actions import DebugError, RunCode, WriteTest from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( diff --git a/metagpt/schema.py b/metagpt/schema.py index 25281e399..baed5582b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -97,14 +97,14 @@ class Message(BaseModel): send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) def __init__( - self, - content, - instruct_content=None, - role="user", - cause_by="", - sent_from="", - send_to=MESSAGE_ROUTE_TO_ALL, - **kwargs, + self, + content, + instruct_content=None, + role="user", + cause_by="", + sent_from="", + send_to=MESSAGE_ROUTE_TO_ALL, + **kwargs, ): """ Parameters not listed below will be stored as meta info, including custom parameters. diff --git a/metagpt/utils/ahttp_client.py b/metagpt/utils/ahttp_client.py index d4f9f94e5..b4a33e9d7 100644 --- a/metagpt/utils/ahttp_client.py +++ b/metagpt/utils/ahttp_client.py @@ -2,29 +2,24 @@ # -*- coding: utf-8 -*- # @Desc : pure async http_client -from typing import Optional, Any, Mapping, Union +from typing import Any, Mapping, Optional, Union -from aiohttp.client import DEFAULT_TIMEOUT import aiohttp +from aiohttp.client import DEFAULT_TIMEOUT -async def apost(url: str, - params: Optional[Mapping[str, str]] = None, - json: Any = None, - data: Any = None, - headers: Optional[dict] = None, - as_json: bool = False, - encoding: str = "utf-8", - timeout: int = DEFAULT_TIMEOUT.total) -> Union[str, dict]: +async def apost( + url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + as_json: bool = False, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total, +) -> Union[str, dict]: async with aiohttp.ClientSession() as session: - async with session.post( - url=url, - params=params, - json=json, - data=data, - headers=headers, - timeout=timeout - ) as resp: + async with session.post(url=url, params=params, json=json, data=data, headers=headers, timeout=timeout) as resp: if as_json: data = await resp.json() else: @@ -33,13 +28,15 @@ async def apost(url: str, return data -async def apost_stream(url: str, - params: Optional[Mapping[str, str]] = None, - json: Any = None, - data: Any = None, - headers: Optional[dict] = None, - encoding: str = "utf-8", - timeout: int = DEFAULT_TIMEOUT.total) -> Any: +async def apost_stream( + url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total, +) -> Any: """ usage: result = astream(url="xx") @@ -47,13 +44,6 @@ async def apost_stream(url: str, deal_with(line) """ async with aiohttp.ClientSession() as session: - async with session.post( - url=url, - params=params, - json=json, - data=data, - headers=headers, - timeout=timeout - ) as resp: + async with session.post(url=url, params=params, json=json, data=data, headers=headers, timeout=timeout) as resp: async for line in resp.content: yield line.decode(encoding) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 9827b8252..1340b1768 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,13 +8,15 @@ """ from __future__ import annotations -from gitignore_parser import parse_gitignore, rule_from_pattern, handle_negation import shutil from enum import Enum from pathlib import Path from typing import Dict, List + from git.repo import Repo from git.repo.fun import is_git_dir +from gitignore_parser import parse_gitignore + from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile @@ -236,8 +238,9 @@ def get_files(self, relative_path: Path | str, root_relative_path: Path | str = rpath = file_path.relative_to(root_relative_path) files.append(str(rpath)) else: - subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path, - filter_ignored=False) + subfolder_files = self.get_files( + relative_path=file_path, root_relative_path=root_relative_path, filter_ignored=False + ) files.extend(subfolder_files) except Exception as e: logger.error(f"Error: {e}") diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index 0a461d360..4aafd8e66 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -4,12 +4,13 @@ import copy from enum import Enum -from typing import Union, Callable +from typing import Callable, Union + import regex as re -from tenacity import retry, stop_after_attempt, wait_fixed, after_log, RetryCallState +from tenacity import RetryCallState, retry, stop_after_attempt, wait_fixed -from metagpt.logs import logger from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.utils.custom_decoder import CustomDecoder @@ -33,7 +34,7 @@ def repair_case_sensitivity(output: str, req_key: str) -> str: if req_key_lower in output_lower: # find the sub-part index, and replace it with raw req_key lidx = output_lower.find(req_key_lower) - source = output[lidx: lidx + len(req_key_lower)] + source = output[lidx : lidx + len(req_key_lower)] output = output.replace(source, req_key) logger.info(f"repair_case_sensitivity: {req_key}") @@ -73,7 +74,7 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - sc = "/" # special char if req_key.startswith("[") and req_key.endswith("]"): if sc in req_key: - left_key = req_key.replace(sc, "") # `[/req_key]` -> `[req_key]` + left_key = req_key.replace(sc, "") # `[/req_key]` -> `[req_key]` right_key = req_key else: left_key = req_key @@ -82,6 +83,7 @@ def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") - if left_key not in output: output = left_key + "\n" + output if right_key not in output: + def judge_potential_json(routput: str, left_key: str) -> Union[str, None]: ridx = routput.rfind(left_key) if ridx < 0: @@ -90,7 +92,7 @@ def judge_potential_json(routput: str, left_key: str) -> Union[str, None]: idx1 = sub_output.rfind("}") idx2 = sub_output.rindex("]") idx = idx1 if idx1 >= idx2 else idx2 - sub_output = sub_output[: idx+1] + sub_output = sub_output[: idx + 1] return sub_output if output.strip().endswith("}") or (output.strip().endswith("]") and not output.strip().endswith(left_key)): @@ -155,9 +157,7 @@ def repair_llm_raw_output(output: str, req_keys: list[str], repair_type: RepairT # do the repairation usually for non-openai models for req_key in req_keys: - output = _repair_llm_raw_output(output=output, - req_key=req_key, - repair_type=repair_type) + output = _repair_llm_raw_output(output=output, req_key=req_key, repair_type=repair_type) return output @@ -187,7 +187,7 @@ def repair_invalid_json(output: str, error: str) -> str: new_line = line.replace("}", "") elif line.endswith("},") and output.endswith("},"): new_line = line[:-1] - elif '",' not in line and ',' not in line: + elif '",' not in line and "," not in line: new_line = f'{line}",' elif "," not in line: # problem, miss char `,` at the end. @@ -228,8 +228,10 @@ def run_and_passon(retry_state: RetryCallState) -> None: elif retry_state.kwargs: func_param_output = retry_state.kwargs.get("output", "") exp_str = str(retry_state.outcome.exception()) - logger.warning(f"parse json from content inside [CONTENT][/CONTENT] failed at retry " - f"{retry_state.attempt_number}, try to fix it, exp: {exp_str}") + logger.warning( + f"parse json from content inside [CONTENT][/CONTENT] failed at retry " + f"{retry_state.attempt_number}, try to fix it, exp: {exp_str}" + ) repaired_output = repair_invalid_json(func_param_output, exp_str) retry_state.kwargs["output"] = repaired_output @@ -260,7 +262,8 @@ def retry_parse_json_text(output: str) -> Union[list, dict]: def extract_content_from_output(content: str, right_key: str = "[/CONTENT]"): - """ extract xxx from [CONTENT](xxx)[/CONTENT] using regex pattern """ + """extract xxx from [CONTENT](xxx)[/CONTENT] using regex pattern""" + def re_extract_content(cont: str, pattern: str) -> str: matches = re.findall(pattern, cont, re.DOTALL) for match in matches: diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py index f479ec3b8..5ceed65d9 100644 --- a/metagpt/utils/utils.py +++ b/metagpt/utils/utils.py @@ -4,7 +4,7 @@ import typing -from tenacity import after_log, _utils +from tenacity import _utils def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: @@ -13,7 +13,10 @@ def log_it(retry_state: "RetryCallState") -> None: fn_name = "" else: fn_name = _utils.get_callback_name(retry_state.fn) - logger.error(f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " - f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " - f"exp: {retry_state.outcome.exception()}") + logger.error( + f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " + f"exp: {retry_state.outcome.exception()}" + ) + return log_it diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 49969a2af..408fd3162 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -33,5 +33,6 @@ async def test_llm_acompletion(llm): assert len(await llm.acompletion_batch([hello_msg])) > 0 assert len(await llm.acompletion_batch_text([hello_msg])) > 0 + # if __name__ == "__main__": # pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_ahttp_client.py b/tests/metagpt/utils/test_ahttp_client.py index 15159423a..a595d645f 100644 --- a/tests/metagpt/utils/test_ahttp_client.py +++ b/tests/metagpt/utils/test_ahttp_client.py @@ -9,30 +9,21 @@ @pytest.mark.asyncio async def test_apost(): - result = await apost( - url="https://www.baidu.com/" - ) + result = await apost(url="https://www.baidu.com/") assert "百度一下" in result result = await apost( - url="http://aider.meizu.com/app/weather/listWeather", - data={"cityIds": "101240101"}, - as_json=True + url="http://aider.meizu.com/app/weather/listWeather", data={"cityIds": "101240101"}, as_json=True ) assert result["code"] == "200" @pytest.mark.asyncio async def test_apost_stream(): - result = apost_stream( - url="https://www.baidu.com/" - ) + result = apost_stream(url="https://www.baidu.com/") async for line in result: assert len(line) >= 0 - result = apost_stream( - url="http://aider.meizu.com/app/weather/listWeather", - data={"cityIds": "101240101"} - ) + result = apost_stream(url="http://aider.meizu.com/app/weather/listWeather", data={"cityIds": "101240101"}) async for line in result: assert len(line) >= 0 diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py index a2dd18516..21bbee921 100644 --- a/tests/metagpt/utils/test_repair_llm_raw_output.py +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -4,10 +4,15 @@ from metagpt.config import CONFIG -CONFIG.repair_llm_output = True +from metagpt.utils.repair_llm_raw_output import ( + RepairType, + extract_content_from_output, + repair_invalid_json, + repair_llm_raw_output, + retry_parse_json_text, +) -from metagpt.utils.repair_llm_raw_output import repair_llm_raw_output, RepairType, repair_invalid_json,\ - extract_content_from_output, retry_parse_json_text +CONFIG.repair_llm_output = True def test_repair_case_sensitivity(): @@ -26,8 +31,7 @@ def test_repair_case_sensitivity(): "Requirement Analysis": "The 2048 game should be simple to play" }""" req_keys = ["Original Requirements", "Search Information", "Competitive Quadrant Chart", "Requirement Analysis"] - output = repair_llm_raw_output(output=raw_output, - req_keys=req_keys) + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) assert output == target_output @@ -40,8 +44,7 @@ def test_repair_special_character_missing(): "Anything UNCLEAR": "No unclear requirements or information." [/CONTENT]""" req_keys = ["[/CONTENT]"] - output = repair_llm_raw_output(output=raw_output, - req_keys=req_keys) + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) assert output == target_output raw_output = """[CONTENT] tag @@ -56,15 +59,13 @@ def test_repair_special_character_missing(): "Anything UNCLEAR": "No unclear requirements or information." } [/CONTENT]""" - output = repair_llm_raw_output(output=raw_output, - req_keys=req_keys) + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) assert output == target_output raw_output = '[CONTENT] {"a": "b"} [CONTENT]' target_output = '[CONTENT] {"a": "b"} [/CONTENT]' - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) print("output\n", output) assert output == target_output @@ -73,38 +74,35 @@ def test_required_key_pair_missing(): raw_output = '[CONTENT] {"a": "b"}' target_output = '[CONTENT] {"a": "b"}\n[/CONTENT]' - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output - raw_output = '''[CONTENT] + raw_output = """[CONTENT] { "key": "value" -]''' - target_output = '''[CONTENT] +]""" + target_output = """[CONTENT] { "key": "value" ] -[/CONTENT]''' +[/CONTENT]""" - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output - raw_output = '''[CONTENT] tag + raw_output = """[CONTENT] tag [CONTENT] { "key": "value" } xxx -''' - target_output = '''[CONTENT] +""" + target_output = """[CONTENT] { "key": "value" } -[/CONTENT]''' - output = repair_llm_raw_output(output=raw_output, - req_keys=["[/CONTENT]"]) +[/CONTENT]""" + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) assert output == target_output @@ -112,25 +110,19 @@ def test_repair_json_format(): raw_output = "{ xxx }]" target_output = "{ xxx }" - output = repair_llm_raw_output(output=raw_output, - req_keys=[None], - repair_type=RepairType.JSON) + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) assert output == target_output raw_output = "[{ xxx }" target_output = "{ xxx }" - output = repair_llm_raw_output(output=raw_output, - req_keys=[None], - repair_type=RepairType.JSON) + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) assert output == target_output raw_output = "{ xxx ]" target_output = "{ xxx }" - output = repair_llm_raw_output(output=raw_output, - req_keys=[None], - repair_type=RepairType.JSON) + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) assert output == target_output @@ -186,7 +178,7 @@ def test_retry_parse_json_text(): target_json = { "Original Requirements": "Create a 2048 game", "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", - "Requirement Analysis": "The requirements are clear and well-defined" + "Requirement Analysis": "The requirements are clear and well-defined", } output = retry_parse_json_text(output=invalid_json_text) assert output == target_json @@ -200,7 +192,7 @@ def test_retry_parse_json_text(): target_json = { "Original Requirements": "Create a 2048 game", "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", - "Requirement Analysis": "The requirements are clear and well-defined" + "Requirement Analysis": "The requirements are clear and well-defined", } output = retry_parse_json_text(output=invalid_json_text) assert output == target_json @@ -214,84 +206,88 @@ def test_extract_content_from_output(): xxx [CONTENT] xxxx [/CONTENT] xxx [CONTENT][/CONTENT] xxx [CONTENT][/CONTENT] # target pair is the last one """ - output = 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' \ - 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' \ - 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' \ - '3.0.0\n\ndescription: A JSON object representing the game state.\n\npaths:\n game:\n get:\n ' \ - 'summary: Get the current game state.\n responses:\n 200:\n description: Game state.' \ - '\n\n moves:\n post:\n summary: Make a move.\n requestBody:\n description: Move to be ' \ - 'made.\n content:\n applicationjson:\n schema:\n type: object\n ' \ - ' properties:\n x:\n type: integer\n y:\n ' \ - ' type: integer\n tile:\n type: object\n ' \ - 'properties:\n value:\n type: integer\n x:\n ' \ - ' type: integer\n y:\n type: integer\n\n ' \ - 'undo-move:\n post:\n summary: Undo the last move.\n responses:\n 200:\n ' \ - ' description: Undone move.\n\n end-game:\n post:\n summary: End the game.\n responses:\n ' \ - ' 200:\n description: Game ended.\n\n start-game:\n post:\n summary: Start a new ' \ - 'game.\n responses:\n 200:\n description: Game started.\n\n game-over:\n get:\n ' \ - ' summary: Check if the game is over.\n responses:\n 200:\n description: Game ' \ - 'over.\n 404:\n description: Game not over.\n\n score:\n get:\n summary: Get the ' \ - 'current score.\n responses:\n 200:\n description: Score.\n\n tile:\n get:\n ' \ - 'summary: Get a specific tile.\n parameters:\n tile_id:\n type: integer\n ' \ - 'description: ID of the tile to get.\n responses:\n 200:\n description: Tile.\n\n ' \ - 'tiles:\n get:\n summary: Get all tiles.\n responses:\n 200:\n description: ' \ - 'Tiles.\n\n level:\n get:\n summary: Get the current level.\n responses:\n 200:\n ' \ - ' description: Level.\n\n level-up:\n post:\n summary: Level up.\n responses:\n ' \ - '200:\n description: Level up successful.\n\n level-down:\n post:\n summary: Level ' \ - 'down.\n responses:\n 200:\n description: Level down successful.\n\n restart:\n ' \ - 'post:\n summary: Restart the game.\n responses:\n 200:\n description: Game ' \ - 'restarted.\n\n help:\n get:\n summary: Get help.\n responses:\n 200:\n ' \ - 'description: Help.\n\n version:\n get:\n summary: Get the version of the game.\n ' \ - 'responses:\n 200:\n description: Version.\n\n}\n\n"Logic Analysis": [\n"game.py",' \ - '\n"Contains the game logic."\n],\n"Task list": [\n"game.py",\n"Contains the game logic and should be ' \ - 'done first."\n],\n"Shared Knowledge": "\n\'game.py\' contains the game logic.\n",\n"Anything ' \ - 'UNCLEAR": "How to start the game."\n]\n\n[/CONTENT] Great! Your JSON output is properly formatted ' \ - 'and correctly includes all the required sections. Here\'s a breakdown of what each section ' \ - 'contains:\n\nRequired Python third-party packages:\n\n* pygame==2.0.4\n* pytest\n\nRequired Other ' \ - 'language third-party packages:\n\n* No third-party packages are required.\n\nFull API spec:\n\n* ' \ - 'openapi: 3.0.0\n* description: A JSON object representing the game state.\n* paths:\n + game: ' \ - 'Get the current game state.\n + moves: Make a move.\n + undo-move: Undo the last move.\n + ' \ - 'end-game: End the game.\n + start-game: Start a new game.\n + game-over: Check if the game is ' \ - 'over.\n + score: Get the current score.\n + tile: Get a specific tile.\n + tiles: Get all tiles.\n ' \ - '+ level: Get the current level.\n + level-up: Level up.\n + level-down: Level down.\n + restart: ' \ - 'Restart the game.\n + help: Get help.\n + version: Get the version of the game.\n\nLogic ' \ - 'Analysis:\n\n* game.py contains the game logic.\n\nTask list:\n\n* game.py contains the game logic ' \ - 'and should be done first.\n\nShared Knowledge:\n\n* \'game.py\' contains the game logic.\n\nAnything ' \ - 'UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and ' \ - 'comprehensive overview of the project\'s requirements and dependencies.' + output = ( + 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' + 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' + 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' + "3.0.0\n\ndescription: A JSON object representing the game state.\n\npaths:\n game:\n get:\n " + "summary: Get the current game state.\n responses:\n 200:\n description: Game state." + "\n\n moves:\n post:\n summary: Make a move.\n requestBody:\n description: Move to be " + "made.\n content:\n applicationjson:\n schema:\n type: object\n " + " properties:\n x:\n type: integer\n y:\n " + " type: integer\n tile:\n type: object\n " + "properties:\n value:\n type: integer\n x:\n " + " type: integer\n y:\n type: integer\n\n " + "undo-move:\n post:\n summary: Undo the last move.\n responses:\n 200:\n " + " description: Undone move.\n\n end-game:\n post:\n summary: End the game.\n responses:\n " + " 200:\n description: Game ended.\n\n start-game:\n post:\n summary: Start a new " + "game.\n responses:\n 200:\n description: Game started.\n\n game-over:\n get:\n " + " summary: Check if the game is over.\n responses:\n 200:\n description: Game " + "over.\n 404:\n description: Game not over.\n\n score:\n get:\n summary: Get the " + "current score.\n responses:\n 200:\n description: Score.\n\n tile:\n get:\n " + "summary: Get a specific tile.\n parameters:\n tile_id:\n type: integer\n " + "description: ID of the tile to get.\n responses:\n 200:\n description: Tile.\n\n " + "tiles:\n get:\n summary: Get all tiles.\n responses:\n 200:\n description: " + "Tiles.\n\n level:\n get:\n summary: Get the current level.\n responses:\n 200:\n " + " description: Level.\n\n level-up:\n post:\n summary: Level up.\n responses:\n " + "200:\n description: Level up successful.\n\n level-down:\n post:\n summary: Level " + "down.\n responses:\n 200:\n description: Level down successful.\n\n restart:\n " + "post:\n summary: Restart the game.\n responses:\n 200:\n description: Game " + "restarted.\n\n help:\n get:\n summary: Get help.\n responses:\n 200:\n " + "description: Help.\n\n version:\n get:\n summary: Get the version of the game.\n " + 'responses:\n 200:\n description: Version.\n\n}\n\n"Logic Analysis": [\n"game.py",' + '\n"Contains the game logic."\n],\n"Task list": [\n"game.py",\n"Contains the game logic and should be ' + 'done first."\n],\n"Shared Knowledge": "\n\'game.py\' contains the game logic.\n",\n"Anything ' + 'UNCLEAR": "How to start the game."\n]\n\n[/CONTENT] Great! Your JSON output is properly formatted ' + "and correctly includes all the required sections. Here's a breakdown of what each section " + "contains:\n\nRequired Python third-party packages:\n\n* pygame==2.0.4\n* pytest\n\nRequired Other " + "language third-party packages:\n\n* No third-party packages are required.\n\nFull API spec:\n\n* " + "openapi: 3.0.0\n* description: A JSON object representing the game state.\n* paths:\n + game: " + "Get the current game state.\n + moves: Make a move.\n + undo-move: Undo the last move.\n + " + "end-game: End the game.\n + start-game: Start a new game.\n + game-over: Check if the game is " + "over.\n + score: Get the current score.\n + tile: Get a specific tile.\n + tiles: Get all tiles.\n " + "+ level: Get the current level.\n + level-up: Level up.\n + level-down: Level down.\n + restart: " + "Restart the game.\n + help: Get help.\n + version: Get the version of the game.\n\nLogic " + "Analysis:\n\n* game.py contains the game logic.\n\nTask list:\n\n* game.py contains the game logic " + "and should be done first.\n\nShared Knowledge:\n\n* 'game.py' contains the game logic.\n\nAnything " + "UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and " + "comprehensive overview of the project's requirements and dependencies." + ) output = extract_content_from_output(output) - assert output.startswith('{\n"Required Python third-party packages') and \ - output.endswith('UNCLEAR": "How to start the game."\n]') - - output = 'Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object ' \ - 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' \ - 'Information": "Search results for 2048 game",\n"Requirements": [\n"Create a game with the same rules ' \ - 'as the original 2048 game",\n"Implement a user interface that is easy to use and understand",\n"Add a ' \ - 'scoreboard to track the player progress",\n"Allow the player to undo and redo moves",\n"Implement a ' \ - 'game over screen to display the final score"\n],\n"Product Goals": [\n"Create a fun and engaging game ' \ - 'experience for the player",\n"Design a user interface that is visually appealing and easy to use",\n"' \ - 'Optimize the game for performance and responsiveness"\n],\n"User Stories": [\n"As a player, I want to ' \ - 'be able to move tiles around the board to combine numbers",\n"As a player, I want to be able to undo ' \ - 'and redo moves to correct mistakes",\n"As a player, I want to see the final score and game over screen' \ - ' when I win"\n],\n"Competitive Analysis": [\n"Competitor A: 2048 game with a simple user interface and' \ - ' basic graphics",\n"Competitor B: 2048 game with a more complex user interface and better graphics",' \ - '\n"Competitor C: 2048 game with a unique twist on the rules and a more challenging gameplay experience"' \ - '\n],\n"Competitive Quadrant Chart": "quadrantChart\\n\ttitle Reach and engagement of campaigns\\n\t\t' \ - 'x-axis Low Reach --> High Reach\\n\t\ty-axis Low Engagement --> High Engagement\\n\tquadrant-1 We ' \ - 'should expand\\n\tquadrant-2 Need to promote\\n\tquadrant-3 Re-evaluate\\n\tquadrant-4 May be ' \ - 'improved\\n\tCampaign A: [0.3, 0.6]\\n\tCampaign B: [0.45, 0.23]\\n\tCampaign C: [0.57, 0.69]\\n\t' \ - 'Campaign D: [0.78, 0.34]\\n\tCampaign E: [0.40, 0.34]\\n\tCampaign F: [0.35, 0.78]"\n],\n"Requirement ' \ - 'Analysis": "The requirements are clear and well-defined, but there may be some ambiguity around the ' \ - 'specific implementation details",\n"Requirement Pool": [\n["P0", "Implement a game with the same ' \ - 'rules as the original 2048 game"],\n["P1", "Add a scoreboard to track the player progress"],\n["P2", ' \ - '"Allow the player to undo and redo moves"]\n],\n"UI Design draft": "The UI should be simple and easy ' \ - 'to use, with a clean and visually appealing design. The game board should be the main focus of the ' \ - 'UI, with clear and concise buttons for the player to interact with.",\n"Anything UNCLEAR": ""\n}\n' \ - '[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything ' \ - 'else I can do to assist you.' + assert output.startswith('{\n"Required Python third-party packages') and output.endswith( + 'UNCLEAR": "How to start the game."\n]' + ) + + output = ( + "Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object " + 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' + 'Information": "Search results for 2048 game",\n"Requirements": [\n"Create a game with the same rules ' + 'as the original 2048 game",\n"Implement a user interface that is easy to use and understand",\n"Add a ' + 'scoreboard to track the player progress",\n"Allow the player to undo and redo moves",\n"Implement a ' + 'game over screen to display the final score"\n],\n"Product Goals": [\n"Create a fun and engaging game ' + 'experience for the player",\n"Design a user interface that is visually appealing and easy to use",\n"' + 'Optimize the game for performance and responsiveness"\n],\n"User Stories": [\n"As a player, I want to ' + 'be able to move tiles around the board to combine numbers",\n"As a player, I want to be able to undo ' + 'and redo moves to correct mistakes",\n"As a player, I want to see the final score and game over screen' + ' when I win"\n],\n"Competitive Analysis": [\n"Competitor A: 2048 game with a simple user interface and' + ' basic graphics",\n"Competitor B: 2048 game with a more complex user interface and better graphics",' + '\n"Competitor C: 2048 game with a unique twist on the rules and a more challenging gameplay experience"' + '\n],\n"Competitive Quadrant Chart": "quadrantChart\\n\ttitle Reach and engagement of campaigns\\n\t\t' + "x-axis Low Reach --> High Reach\\n\t\ty-axis Low Engagement --> High Engagement\\n\tquadrant-1 We " + "should expand\\n\tquadrant-2 Need to promote\\n\tquadrant-3 Re-evaluate\\n\tquadrant-4 May be " + "improved\\n\tCampaign A: [0.3, 0.6]\\n\tCampaign B: [0.45, 0.23]\\n\tCampaign C: [0.57, 0.69]\\n\t" + 'Campaign D: [0.78, 0.34]\\n\tCampaign E: [0.40, 0.34]\\n\tCampaign F: [0.35, 0.78]"\n],\n"Requirement ' + 'Analysis": "The requirements are clear and well-defined, but there may be some ambiguity around the ' + 'specific implementation details",\n"Requirement Pool": [\n["P0", "Implement a game with the same ' + 'rules as the original 2048 game"],\n["P1", "Add a scoreboard to track the player progress"],\n["P2", ' + '"Allow the player to undo and redo moves"]\n],\n"UI Design draft": "The UI should be simple and easy ' + "to use, with a clean and visually appealing design. The game board should be the main focus of the " + 'UI, with clear and concise buttons for the player to interact with.",\n"Anything UNCLEAR": ""\n}\n' + "[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything " + "else I can do to assist you." + ) output = extract_content_from_output(output) - assert output.startswith('{\n"Original Requirements"') and \ - output.endswith('"Anything UNCLEAR": ""\n}') + assert output.startswith('{\n"Original Requirements"') and output.endswith('"Anything UNCLEAR": ""\n}') output = """ Sure, I'd be happy to help! Here's the JSON output for the given context:\n\n[CONTENT]\n{ "Implementation approach": "We will use the open-source framework PyGame to create a 2D game engine, which will @@ -316,5 +312,6 @@ class Game{\n +int score\n +list tiles\n +function move_ information for a developer to understand the design and implementation of the 2048 game. """ output = extract_content_from_output(output) - assert output.startswith('{\n"Implementation approach"') and \ - output.endswith('"Anything UNCLEAR": "The requirement is clear to me."\n}') + assert output.startswith('{\n"Implementation approach"') and output.endswith( + '"Anything UNCLEAR": "The requirement is clear to me."\n}' + ) From ce1895a40bfde64af82d6a5cde5c90c1fcef41b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 14 Dec 2023 21:28:11 +0800 Subject: [PATCH 219/232] feat: Assume it's new requirements if the code directory does not exist --- metagpt/actions/write_prd.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index b9bad2233..bb0cf8fb9 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -190,5 +190,9 @@ async def _rename_workspace(prd): CONFIG.git_repo.rename_root(CONFIG.project_name) async def _is_bugfix(self, context) -> bool: + src_workspace_path = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name + code_files = CONFIG.git_repo.get_files(relative_path=src_workspace_path) + if not code_files: + return False node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm) return node.get("issue_type") == "BUG" From 84357651e53a82669238ae91ed98610810ddcd89 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 14 Dec 2023 23:54:00 +0800 Subject: [PATCH 220/232] resolve conflicts --- metagpt/actions/action_node.py | 4 ---- metagpt/actions/project_management_an.py | 3 ++- metagpt/actions/write_prd_an.py | 2 +- metagpt/roles/architect.py | 3 ++- metagpt/roles/engineer.py | 3 ++- metagpt/roles/product_manager.py | 4 ++-- metagpt/roles/project_manager.py | 4 ++-- metagpt/team.py | 2 +- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index f5009f345..9fb10f35c 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -52,10 +52,6 @@ def dict_to_markdown(d, prefix="-", postfix="\n"): class ActionNode: """ActionNode is a tree of nodes.""" - - # Action Strgy - # - sop: 仅使用一级SOP - # - complex: 使用一级SOP+自定义策略填槽 mode: str # Action Context diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py index e03af36d7..970cb0594 100644 --- a/metagpt/actions/project_management_an.py +++ b/metagpt/actions/project_management_an.py @@ -43,7 +43,8 @@ FULL_API_SPEC = ActionNode( key="Full API spec", expected_type=str, - instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend.", + instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend. If front-end " + "and back-end communication is not required, leave it blank.", example="openapi: 3.0.0 ...", ) diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index 2c81bdb6e..68402e504 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -86,7 +86,7 @@ key="Requirement Analysis", expected_type=str, instruction="Provide a detailed analysis of the requirements.", - example="The product should be user-friendly and performance-optimized.", + example="The product should be user-friendly.", ) REQUIREMENT_POOL = ActionNode( diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 2c0bdd1d6..fa91d393d 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -27,7 +27,8 @@ def __init__( name: str = "Bob", profile: str = "Architect", goal: str = "design a concise, usable, complete software system", - constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries", + constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries." + "Use same language as user requirement" ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 844f3589d..2f99d132e 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -72,7 +72,8 @@ def __init__( name: str = "Alex", profile: str = "Engineer", goal: str = "write elegant, readable, extensible, efficient code", - constraints: str = "the code should conform to standards like PEP8 and be modular and maintainable", + constraints: str = "the code should conform to standards like PEP8 and be modular and maintainable. " + "Use same language as user requirement", n_borg: int = 1, use_code_review: bool = False, ) -> None: diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 017feade7..e5e9f2b5e 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -28,8 +28,8 @@ def __init__( self, name: str = "Alice", profile: str = "Product Manager", - goal: str = "Efficiently create a successful product", - constraints: str = "", + goal: str = "efficiently create a successful product", + constraints: str = "use same language as user requirement", ) -> None: """ Initializes the ProductManager role with given attributes. diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index bfe1be251..5a2b9be50 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -26,8 +26,8 @@ def __init__( name: str = "Eve", profile: str = "Project Manager", goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task " - "dependencies to start with the prerequisite modules", - constraints: str = "", + "dependencies to start with the prerequisite modules", + constraints: str = "use same language as user requirement", ) -> None: """ Initializes the ProjectManager role with given attributes. diff --git a/metagpt/team.py b/metagpt/team.py index 92f379c97..e1b2a9ffc 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -63,7 +63,7 @@ async def run(self, n_round=3): while n_round > 0: # self._save() n_round -= 1 - logger.debug(f"{n_round=}") + logger.info(f"max {n_round=} left.") self._check_balance() await self.env.run() if CONFIG.git_repo: From ad0ac940936e089058842f953426b25533d7614f Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 14 Dec 2023 20:27:18 +0800 Subject: [PATCH 221/232] fix code review performance drop --- metagpt/actions/write_code.py | 6 ++++-- metagpt/actions/write_code_review.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index a2501db2a..b759f4e2a 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -115,7 +115,7 @@ async def run(self, *args, **kwargs) -> CodingContext: if test_doc: test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr - code_context = await self._get_codes(coding_context.task_doc) + code_context = await self.get_codes(coding_context.task_doc, exclude=self.context.filename) prompt = PROMPT_TEMPLATE.format( design=coding_context.design_doc.content, tasks=coding_context.task_doc.content if coding_context.task_doc else "", @@ -133,7 +133,7 @@ async def run(self, *args, **kwargs) -> CodingContext: return coding_context @staticmethod - async def _get_codes(task_doc) -> str: + async def get_codes(task_doc, exclude) -> str: if not task_doc: return "" if not task_doc.content: @@ -143,6 +143,8 @@ async def _get_codes(task_doc) -> str: codes = [] src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) for filename in code_filenames: + if filename == exclude: + continue doc = await src_file_repo.get(filename=filename) if not doc: continue diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index e0a538fc8..75313fea5 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -10,6 +10,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential +from metagpt.actions import WriteCode from metagpt.actions.action import Action from metagpt.config import CONFIG from metagpt.logs import logger @@ -109,11 +110,12 @@ async def run(self, *args, **kwargs) -> CodingContext: for i in range(k): format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) task_content = self.context.task_doc.content if self.context.task_doc else "" + code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename) context = "\n----------\n".join( [ "```text\n" + self.context.design_doc.content + "```\n", "```text\n" + task_content + "```\n", - "```python\n" + self.context.code_doc.content + "```\n", + "```python\n" + code_context + "```\n", ] ) prompt = PROMPT_TEMPLATE.format( From ccecb45b13f5786c5ff842ee27516f67ec97b7f4 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 14 Dec 2023 23:54:38 +0800 Subject: [PATCH 222/232] resolve conflicts --- metagpt/actions/action.py | 1 + metagpt/actions/action_node.py | 2 +- metagpt/actions/write_code.py | 63 +++++++++---------- metagpt/actions/write_code_review.py | 93 ++++++++++++++++------------ metagpt/actions/write_prd_an.py | 2 +- metagpt/provider/base_gpt_api.py | 2 +- metagpt/roles/engineer.py | 7 ++- metagpt/roles/role.py | 6 +- metagpt/schema.py | 2 +- 9 files changed, 95 insertions(+), 83 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 1d9be60e0..6c1f63f45 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -44,6 +44,7 @@ def set_prefix(self, prefix, profile): self.prefix = prefix self.profile = profile self.llm.system_prompt = prefix + return self def __str__(self): return self.__class__.__name__ diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 9fb10f35c..1d808ec70 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -243,7 +243,7 @@ def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) ) return prompt - @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) + @retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(6)) async def _aask_v1( self, prompt: str, diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index b759f4e2a..a91e4ee1e 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -34,59 +34,52 @@ PROMPT_TEMPLATE = """ NOTICE -Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) +Role: You are a professional engineer; the main goal is to write google-style, elegant, modular, easy to read and maintain code Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ------ -# Design -```json +# Context +## Design {design} -``` ------ -# Tasks -```json + +## Tasks {tasks} -``` ------ -# Legacy Code -```python + +## Legacy Code +```Code {code} ``` ------ -# Debug logs + +## Debug logs ```text {logs} {summary_log} ``` ------ -# Bug Feedback logs + +## Bug Feedback logs ```text {feedback} ``` ------ - - -## Code: {filename} Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. -8. Before using a variable, make sure you reference it first -9. Write out EVERY DETAIL, DON'T LEAVE TODO. - -## Format example ------ + +# Format example ## Code: {filename} ```python ## {filename} ... ``` ------ + +# Instruction: Based on the context, follow "Format example", write code. + +## Code: {filename} Write code with triple quoto, based on the following attentions and context. +1. Only One file: do your best to implement THIS ONLY ONE FILE. +2. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. AVOID circular import. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. +5. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +6. Before using a external variable/module, make sure you import it first. +7. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. + """ @@ -148,5 +141,5 @@ async def get_codes(task_doc, exclude) -> str: doc = await src_file_repo.get(filename=filename) if not doc: continue - codes.append(doc.content) - return "\n----------\n".join(codes) + codes.append(f"----- {filename}\n" + doc.content) + return "\n".join(codes) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 75313fea5..f63a399a9 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -18,8 +18,8 @@ from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ -NOTICE -Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +# System +Role: You are a professional software engineer, and your main task is to review and revise the code. You need to ensure that the code conforms to the google-style standards, is elegantly designed and modularized, easy to read and maintain. Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". @@ -27,53 +27,52 @@ {context} ## Code to be Reviewed: {filename} -``` +```Code {code} ``` +""" + + +EXAMPLE_AND_INSTRUCTION = """ + +{format_example} + ------ +# Instruction: Based on the actual code situation, follow one of the "Format example". -## Code Review: Based on the "Code to be Reviewed", provide key, clear, concise, and specific code modification suggestions, up to 5. +## Code Review: Ordered List. Based on the "Code to be Reviewed", provide key, clear, concise, and specific answer. If any answer is no, explain how to fix it step by step. 1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. 2. Is the code logic completely correct? If there are errors, please indicate how to correct them. 3. Does the existing code follow the "Data structures and interfaces"? 4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step. 5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported -6. Is the code implemented concisely enough? Are methods from other files being reused correctly? +6. Are methods from other files being reused correctly? -## Code Review Result: If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. -LGTM/LBTM - -## Rewrite Code: if it still has some bugs, rewrite {filename} based on "Code Review" with triple quotes, try to get LGTM. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. RETURN ALL CODE, NEVER OMIT ANYTHING. 以任何方式省略代码都是不允许的。 -``` -``` +## Actions: Ordered List. Things that should be done after CR, such as implementing class A and function B -## Format example -{format_example} +## Code Review Result: str. If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. +LGTM/LBTM """ FORMAT_EXAMPLE = """ ------ -# EXAMPLE 1 +# Format example 1 ## Code Review: {filename} -1. No, we should add the logic of ... +1. No, we should fix the logic of class A due to ... 2. ... 3. ... -4. ... +4. No, function B is not implemented, ... 5. ... 6. ... -## Code Review Result: {filename} +## Actions +1. fix class A +2. implement function B + +## Code Review Result LBTM -## Rewrite Code: {filename} -```python -## {filename} -... -``` ------ -# EXAMPLE 2 +# Format example 2 ## Code Review: {filename} 1. Yes. 2. Yes. @@ -82,12 +81,20 @@ 5. Yes. 6. Yes. -## Code Review Result: {filename} +## Actions +pass + +## Code Review Result LGTM +""" -## Rewrite Code: {filename} -pass ------ +REWRITE_CODE_TEMPLATE = """ +# Instruction: rewrite code based on the Code Review and Actions +## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes. +```Code +## {filename} +... +``` """ @@ -96,11 +103,15 @@ def __init__(self, name="WriteCodeReview", context=None, llm=None): super().__init__(name, context, llm) @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) - async def write_code_review_and_rewrite(self, prompt): - code_rsp = await self._aask(prompt) - result = CodeParser.parse_block("Code Review Result", code_rsp) + async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, filename): + cr_rsp = await self._aask(context_prompt + cr_prompt) + result = CodeParser.parse_block("Code Review Result", cr_rsp) if "LGTM" in result: return result, None + + # if LBTM, rewrite code + rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}" + code_rsp = await self._aask(rewrite_prompt) code = CodeParser.parse_code(block="", text=code_rsp) return result, code @@ -111,23 +122,23 @@ async def run(self, *args, **kwargs) -> CodingContext: format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) task_content = self.context.task_doc.content if self.context.task_doc else "" code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename) - context = "\n----------\n".join( + context = "\n".join( [ - "```text\n" + self.context.design_doc.content + "```\n", - "```text\n" + task_content + "```\n", - "```python\n" + code_context + "```\n", + "## System Design\n" + self.context.design_doc.content + "\n", + "## Tasks\n" + task_content + "\n", + "## Code Files\n" + code_context + "\n", ] ) - prompt = PROMPT_TEMPLATE.format( + context_prompt = PROMPT_TEMPLATE.format( context=context, code=iterative_code, filename=self.context.code_doc.filename, - format_example=format_example, ) + cr_prompt = EXAMPLE_AND_INSTRUCTION.format(format_example=format_example, ) logger.info( - f"Code review and rewrite {self.context.code_doc.filename,}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}" + f"Code review and rewrite {self.context.code_doc.filename}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}" ) - result, rewrited_code = await self.write_code_review_and_rewrite(prompt) + result, rewrited_code = await self.write_code_review_and_rewrite(context_prompt, cr_prompt, self.context.code_doc.filename) if "LBTM" in result: iterative_code = rewrited_code elif "LGTM" in result: diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index 68402e504..d96c0aeac 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -93,7 +93,7 @@ key="Requirement Pool", expected_type=list[list[str]], instruction="List down the requirements with their priority (P0, P1, P2).", - example=[["P0", "High priority requirement"], ["P1", "Medium priority requirement"]], + example=[["P0", "..."], ["P1", "..."]], ) UI_DESIGN_DRAFT = ActionNode( diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 6c1dc8338..c38576806 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -49,8 +49,8 @@ async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, stream=T message = ( [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] ) - rsp = await self.acompletion_text(message, stream=stream) logger.debug(message) + rsp = await self.acompletion_text(message, stream=stream) # logger.debug(rsp) return rsp diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 2f99d132e..f1e65b177 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -72,7 +72,7 @@ def __init__( name: str = "Alex", profile: str = "Engineer", goal: str = "write elegant, readable, extensible, efficient code", - constraints: str = "the code should conform to standards like PEP8 and be modular and maintainable. " + constraints: str = "the code should conform to standards like google-style and be modular and maintainable. " "Use same language as user requirement", n_borg: int = 1, use_code_review: bool = False, @@ -105,7 +105,9 @@ async def _act_sp_with_cr(self, review=False) -> Set[str]: coding_context = await todo.run() # Code review if review: - coding_context = await WriteCodeReview(context=coding_context, llm=self._llm).run() + action = WriteCodeReview(context=coding_context, llm=self._llm) + self._init_action_system_message(action) + coding_context = await action.run() await src_file_repo.save( coding_context.filename, dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}, @@ -224,6 +226,7 @@ async def _new_coding_context( task_doc = await task_file_repo.get(i.name) elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: design_doc = await design_file_repo.get(i.name) + # FIXME: design doc没有加载进来，是None context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) return context diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 633ad6051..66475da72 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -134,6 +134,7 @@ def __init__(self, name="", profile="", goal="", constraints="", desc="", is_hum self._setting = RoleSetting( name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human ) + self._llm.system_prompt = self._get_prefix() self._states = [] self._actions = [] self._role_id = str(self._setting) @@ -144,6 +145,9 @@ def _reset(self): self._states = [] self._actions = [] + def _init_action_system_message(self, action: Action): + action.set_prefix(self._get_prefix(), self.profile) + def _init_actions(self, actions): self._reset() for idx, action in enumerate(actions): @@ -158,7 +162,7 @@ def _init_actions(self, actions): ) i = action # i.set_env(self._rc.env) - i.set_prefix(self._get_prefix(), self.profile) + self._init_action_system_message(i) self._actions.append(i) self._states.append(f"{idx}. {action}") diff --git a/metagpt/schema.py b/metagpt/schema.py index baed5582b..799bb9253 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -259,7 +259,7 @@ def load(self, v) -> "MessageQueue": class CodingContext(BaseModel): filename: str - design_doc: Document + design_doc: Optional[Document] task_doc: Optional[Document] code_doc: Optional[Document] From 222694c329d5bddc412317d4e20c774d391776b3 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 00:37:10 +0800 Subject: [PATCH 223/232] fix bugs --- metagpt/actions/write_code.py | 13 +++++++++---- metagpt/actions/write_code_review.py | 2 +- metagpt/schema.py | 6 ++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index a91e4ee1e..5960e2621 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -71,7 +71,7 @@ # Instruction: Based on the context, follow "Format example", write code. -## Code: {filename} Write code with triple quoto, based on the following attentions and context. +## Code: {filename}. Write code with triple quoto, based on the following attentions and context. 1. Only One file: do your best to implement THIS ONLY ONE FILE. 2. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. 3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. AVOID circular import. @@ -100,7 +100,7 @@ async def run(self, *args, **kwargs) -> CodingContext: filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) summary_doc = None - if coding_context.design_doc.filename: + if coding_context.design_doc and coding_context.design_doc.filename: summary_doc = await FileRepository.get_file( filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO ) @@ -108,9 +108,14 @@ async def run(self, *args, **kwargs) -> CodingContext: if test_doc: test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr - code_context = await self.get_codes(coding_context.task_doc, exclude=self.context.filename) + + if bug_feedback: + code_context = coding_context.code_doc.content + else: + code_context = await self.get_codes(coding_context.task_doc, exclude=self.context.filename) + prompt = PROMPT_TEMPLATE.format( - design=coding_context.design_doc.content, + design=coding_context.design_doc.content if coding_context.design_doc else "", tasks=coding_context.task_doc.content if coding_context.task_doc else "", code=code_context, logs=logs, diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index f63a399a9..62e96acd8 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -124,7 +124,7 @@ async def run(self, *args, **kwargs) -> CodingContext: code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename) context = "\n".join( [ - "## System Design\n" + self.context.design_doc.content + "\n", + "## System Design\n" + str(self.context.design_doc) + "\n", "## Tasks\n" + task_content + "\n", "## Code Files\n" + code_context + "\n", ] diff --git a/metagpt/schema.py b/metagpt/schema.py index 799bb9253..758149efa 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -74,6 +74,12 @@ def full_path(self): return None return str(CONFIG.git_repo.workdir / self.root_path / self.filename) + def __str__(self): + return self.content + + def __repr__(self): + return self.content + class Documents(BaseModel): """A class representing a collection of documents. From 126bcdafb966ef694dcf764dc98302bc57497f27 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 10:44:18 +0800 Subject: [PATCH 224/232] fix error msg --- metagpt/utils/git_repository.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 1340b1768..d2bdf5d85 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -233,6 +233,8 @@ def get_files(self, relative_path: Path | str, root_relative_path: Path | str = files = [] try: directory_path = Path(self.workdir) / relative_path + if not directory_path.exists(): + return [] for file_path in directory_path.iterdir(): if file_path.is_file(): rpath = file_path.relative_to(root_relative_path) From 862707d4b7bd319873e550010253a8df0844f6b8 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 10:56:08 +0800 Subject: [PATCH 225/232] use react instead of _react --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 66475da72..b673c330d 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -412,7 +412,7 @@ async def run(self, with_message=None): logger.debug(f"{self._setting}: no news. waiting.") return - rsp = await self._react() + rsp = await self.react() # Reset the next action to be taken. self._rc.todo = None From b97ca3af7ecf980d3ce00675a632c66b9d0989f0 Mon Sep 17 00:00:00 2001 From: geekan Date: Thu, 14 Dec 2023 23:54:38 +0800 Subject: [PATCH 226/232] feat: resolve conflicts --- metagpt/actions/action.py | 1 + metagpt/actions/action_node.py | 2 +- metagpt/actions/project_management.py | 2 +- metagpt/actions/write_code.py | 74 +++++++++++---------- metagpt/actions/write_code_review.py | 93 +++++++++++++++------------ metagpt/actions/write_prd_an.py | 2 +- metagpt/provider/base_gpt_api.py | 2 +- metagpt/roles/engineer.py | 7 +- metagpt/roles/role.py | 8 ++- metagpt/schema.py | 8 ++- metagpt/utils/common.py | 9 ++- metagpt/utils/git_repository.py | 2 + 12 files changed, 120 insertions(+), 90 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 1d9be60e0..6c1f63f45 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -44,6 +44,7 @@ def set_prefix(self, prefix, profile): self.prefix = prefix self.profile = profile self.llm.system_prompt = prefix + return self def __str__(self): return self.__class__.__name__ diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 9fb10f35c..1d808ec70 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -243,7 +243,7 @@ def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) ) return prompt - @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) + @retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(6)) async def _aask_v1( self, prompt: str, diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index c95be4012..1f14e7944 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -99,7 +99,7 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format): async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content) node = await PM_NODE.fill(context, self.llm, format) - task_doc.content = node.content + task_doc.content = node.instruct_content.json(ensure_ascii=False) return task_doc @staticmethod diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index b759f4e2a..5960e2621 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -34,59 +34,52 @@ PROMPT_TEMPLATE = """ NOTICE -Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) +Role: You are a professional engineer; the main goal is to write google-style, elegant, modular, easy to read and maintain code Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ------ -# Design -```json +# Context +## Design {design} -``` ------ -# Tasks -```json + +## Tasks {tasks} -``` ------ -# Legacy Code -```python + +## Legacy Code +```Code {code} ``` ------ -# Debug logs + +## Debug logs ```text {logs} {summary_log} ``` ------ -# Bug Feedback logs + +## Bug Feedback logs ```text {feedback} ``` ------ - - -## Code: {filename} Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. -8. Before using a variable, make sure you reference it first -9. Write out EVERY DETAIL, DON'T LEAVE TODO. - -## Format example ------ + +# Format example ## Code: {filename} ```python ## {filename} ... ``` ------ + +# Instruction: Based on the context, follow "Format example", write code. + +## Code: {filename}. Write code with triple quoto, based on the following attentions and context. +1. Only One file: do your best to implement THIS ONLY ONE FILE. +2. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets. +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. AVOID circular import. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design. +5. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +6. Before using a external variable/module, make sure you import it first. +7. Write out EVERY CODE DETAIL, DON'T LEAVE TODO. + """ @@ -107,7 +100,7 @@ async def run(self, *args, **kwargs) -> CodingContext: filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) summary_doc = None - if coding_context.design_doc.filename: + if coding_context.design_doc and coding_context.design_doc.filename: summary_doc = await FileRepository.get_file( filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO ) @@ -115,9 +108,14 @@ async def run(self, *args, **kwargs) -> CodingContext: if test_doc: test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr - code_context = await self.get_codes(coding_context.task_doc, exclude=self.context.filename) + + if bug_feedback: + code_context = coding_context.code_doc.content + else: + code_context = await self.get_codes(coding_context.task_doc, exclude=self.context.filename) + prompt = PROMPT_TEMPLATE.format( - design=coding_context.design_doc.content, + design=coding_context.design_doc.content if coding_context.design_doc else "", tasks=coding_context.task_doc.content if coding_context.task_doc else "", code=code_context, logs=logs, @@ -148,5 +146,5 @@ async def get_codes(task_doc, exclude) -> str: doc = await src_file_repo.get(filename=filename) if not doc: continue - codes.append(doc.content) - return "\n----------\n".join(codes) + codes.append(f"----- {filename}\n" + doc.content) + return "\n".join(codes) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 75313fea5..62e96acd8 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -18,8 +18,8 @@ from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ -NOTICE -Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +# System +Role: You are a professional software engineer, and your main task is to review and revise the code. You need to ensure that the code conforms to the google-style standards, is elegantly designed and modularized, easy to read and maintain. Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". @@ -27,53 +27,52 @@ {context} ## Code to be Reviewed: {filename} -``` +```Code {code} ``` +""" + + +EXAMPLE_AND_INSTRUCTION = """ + +{format_example} + ------ +# Instruction: Based on the actual code situation, follow one of the "Format example". -## Code Review: Based on the "Code to be Reviewed", provide key, clear, concise, and specific code modification suggestions, up to 5. +## Code Review: Ordered List. Based on the "Code to be Reviewed", provide key, clear, concise, and specific answer. If any answer is no, explain how to fix it step by step. 1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. 2. Is the code logic completely correct? If there are errors, please indicate how to correct them. 3. Does the existing code follow the "Data structures and interfaces"? 4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step. 5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported -6. Is the code implemented concisely enough? Are methods from other files being reused correctly? +6. Are methods from other files being reused correctly? -## Code Review Result: If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. -LGTM/LBTM - -## Rewrite Code: if it still has some bugs, rewrite {filename} based on "Code Review" with triple quotes, try to get LGTM. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. RETURN ALL CODE, NEVER OMIT ANYTHING. 以任何方式省略代码都是不允许的。 -``` -``` +## Actions: Ordered List. Things that should be done after CR, such as implementing class A and function B -## Format example -{format_example} +## Code Review Result: str. If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. +LGTM/LBTM """ FORMAT_EXAMPLE = """ ------ -# EXAMPLE 1 +# Format example 1 ## Code Review: {filename} -1. No, we should add the logic of ... +1. No, we should fix the logic of class A due to ... 2. ... 3. ... -4. ... +4. No, function B is not implemented, ... 5. ... 6. ... -## Code Review Result: {filename} +## Actions +1. fix class A +2. implement function B + +## Code Review Result LBTM -## Rewrite Code: {filename} -```python -## {filename} -... -``` ------ -# EXAMPLE 2 +# Format example 2 ## Code Review: {filename} 1. Yes. 2. Yes. @@ -82,12 +81,20 @@ 5. Yes. 6. Yes. -## Code Review Result: {filename} +## Actions +pass + +## Code Review Result LGTM +""" -## Rewrite Code: {filename} -pass ------ +REWRITE_CODE_TEMPLATE = """ +# Instruction: rewrite code based on the Code Review and Actions +## Rewrite Code: CodeBlock. If it still has some bugs, rewrite {filename} with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Return all completed codes and prohibit the return of unfinished codes. +```Code +## {filename} +... +``` """ @@ -96,11 +103,15 @@ def __init__(self, name="WriteCodeReview", context=None, llm=None): super().__init__(name, context, llm) @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) - async def write_code_review_and_rewrite(self, prompt): - code_rsp = await self._aask(prompt) - result = CodeParser.parse_block("Code Review Result", code_rsp) + async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, filename): + cr_rsp = await self._aask(context_prompt + cr_prompt) + result = CodeParser.parse_block("Code Review Result", cr_rsp) if "LGTM" in result: return result, None + + # if LBTM, rewrite code + rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}" + code_rsp = await self._aask(rewrite_prompt) code = CodeParser.parse_code(block="", text=code_rsp) return result, code @@ -111,23 +122,23 @@ async def run(self, *args, **kwargs) -> CodingContext: format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) task_content = self.context.task_doc.content if self.context.task_doc else "" code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename) - context = "\n----------\n".join( + context = "\n".join( [ - "```text\n" + self.context.design_doc.content + "```\n", - "```text\n" + task_content + "```\n", - "```python\n" + code_context + "```\n", + "## System Design\n" + str(self.context.design_doc) + "\n", + "## Tasks\n" + task_content + "\n", + "## Code Files\n" + code_context + "\n", ] ) - prompt = PROMPT_TEMPLATE.format( + context_prompt = PROMPT_TEMPLATE.format( context=context, code=iterative_code, filename=self.context.code_doc.filename, - format_example=format_example, ) + cr_prompt = EXAMPLE_AND_INSTRUCTION.format(format_example=format_example, ) logger.info( - f"Code review and rewrite {self.context.code_doc.filename,}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}" + f"Code review and rewrite {self.context.code_doc.filename}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}" ) - result, rewrited_code = await self.write_code_review_and_rewrite(prompt) + result, rewrited_code = await self.write_code_review_and_rewrite(context_prompt, cr_prompt, self.context.code_doc.filename) if "LBTM" in result: iterative_code = rewrited_code elif "LGTM" in result: diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index 68402e504..d96c0aeac 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -93,7 +93,7 @@ key="Requirement Pool", expected_type=list[list[str]], instruction="List down the requirements with their priority (P0, P1, P2).", - example=[["P0", "High priority requirement"], ["P1", "Medium priority requirement"]], + example=[["P0", "..."], ["P1", "..."]], ) UI_DESIGN_DRAFT = ActionNode( diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 6c1dc8338..c38576806 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -49,8 +49,8 @@ async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, stream=T message = ( [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] ) - rsp = await self.acompletion_text(message, stream=stream) logger.debug(message) + rsp = await self.acompletion_text(message, stream=stream) # logger.debug(rsp) return rsp diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 2f99d132e..f1e65b177 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -72,7 +72,7 @@ def __init__( name: str = "Alex", profile: str = "Engineer", goal: str = "write elegant, readable, extensible, efficient code", - constraints: str = "the code should conform to standards like PEP8 and be modular and maintainable. " + constraints: str = "the code should conform to standards like google-style and be modular and maintainable. " "Use same language as user requirement", n_borg: int = 1, use_code_review: bool = False, @@ -105,7 +105,9 @@ async def _act_sp_with_cr(self, review=False) -> Set[str]: coding_context = await todo.run() # Code review if review: - coding_context = await WriteCodeReview(context=coding_context, llm=self._llm).run() + action = WriteCodeReview(context=coding_context, llm=self._llm) + self._init_action_system_message(action) + coding_context = await action.run() await src_file_repo.save( coding_context.filename, dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}, @@ -224,6 +226,7 @@ async def _new_coding_context( task_doc = await task_file_repo.get(i.name) elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: design_doc = await design_file_repo.get(i.name) + # FIXME: design doc没有加载进来，是None context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) return context diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 633ad6051..b673c330d 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -134,6 +134,7 @@ def __init__(self, name="", profile="", goal="", constraints="", desc="", is_hum self._setting = RoleSetting( name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human ) + self._llm.system_prompt = self._get_prefix() self._states = [] self._actions = [] self._role_id = str(self._setting) @@ -144,6 +145,9 @@ def _reset(self): self._states = [] self._actions = [] + def _init_action_system_message(self, action: Action): + action.set_prefix(self._get_prefix(), self.profile) + def _init_actions(self, actions): self._reset() for idx, action in enumerate(actions): @@ -158,7 +162,7 @@ def _init_actions(self, actions): ) i = action # i.set_env(self._rc.env) - i.set_prefix(self._get_prefix(), self.profile) + self._init_action_system_message(i) self._actions.append(i) self._states.append(f"{idx}. {action}") @@ -408,7 +412,7 @@ async def run(self, with_message=None): logger.debug(f"{self._setting}: no news. waiting.") return - rsp = await self._react() + rsp = await self.react() # Reset the next action to be taken. self._rc.todo = None diff --git a/metagpt/schema.py b/metagpt/schema.py index baed5582b..758149efa 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -74,6 +74,12 @@ def full_path(self): return None return str(CONFIG.git_repo.workdir / self.root_path / self.filename) + def __str__(self): + return self.content + + def __repr__(self): + return self.content + class Documents(BaseModel): """A class representing a collection of documents. @@ -259,7 +265,7 @@ def load(self, v) -> "MessageQueue": class CodingContext(BaseModel): filename: str - design_doc: Document + design_doc: Optional[Document] task_doc: Optional[Document] code_doc: Optional[Document] diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index f08519f8e..a9bdd6e2d 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -223,10 +223,15 @@ def parse_blocks(cls, text: str): # 遍历所有的block for block in blocks: # 如果block不为空，则继续处理 - if block.strip() != "": + if block.strip() == "": + continue + if "\n" not in block: + block_title = block + block_content = "" + else: # 将block的标题和内容分开，并分别去掉前后的空白字符 block_title, block_content = block.split("\n", 1) - block_dict[block_title.strip()] = block_content.strip() + block_dict[block_title.strip()] = block_content.strip() return block_dict diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 1340b1768..d2bdf5d85 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -233,6 +233,8 @@ def get_files(self, relative_path: Path | str, root_relative_path: Path | str = files = [] try: directory_path = Path(self.workdir) / relative_path + if not directory_path.exists(): + return [] for file_path in directory_path.iterdir(): if file_path.is_file(): rpath = file_path.relative_to(root_relative_path) From 60957372fcb3a810f931443b3a8d7bbcbf1d4e2a Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 11:36:24 +0800 Subject: [PATCH 227/232] tuning log level --- metagpt/config.py | 2 +- metagpt/team.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index d04da1d91..8ad42c99f 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -46,7 +46,7 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._init_with_config_files_and_env(yaml_file) - logger.info("Config loading done.") + logger.debug("Config loading done.") self._update() def _update(self): diff --git a/metagpt/team.py b/metagpt/team.py index e1b2a9ffc..a5c405f80 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -63,7 +63,7 @@ async def run(self, n_round=3): while n_round > 0: # self._save() n_round -= 1 - logger.info(f"max {n_round=} left.") + logger.debug(f"max {n_round=} left.") self._check_balance() await self.env.run() if CONFIG.git_repo: From 3a448a7bb48fefea3a2e377ab42e44a3ddd4deb4 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 11:54:30 +0800 Subject: [PATCH 228/232] config: adjust default values --- config/config.yaml | 10 +++++----- metagpt/config.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index ef8575e43..8fd208c59 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -77,8 +77,8 @@ RPM: 10 #### for Stable Diffusion ## Use SD service, based on https://github.com/AUTOMATIC1111/stable-diffusion-webui -SD_URL: "YOUR_SD_URL" -SD_T2I_API: "/sdapi/v1/txt2img" +#SD_URL: "YOUR_SD_URL" +#SD_T2I_API: "/sdapi/v1/txt2img" #### for Execution #LONG_TERM_MEMORY: false @@ -93,8 +93,8 @@ SD_T2I_API: "/sdapi/v1/txt2img" # CALC_USAGE: false ### for Research -MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo -MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k +# MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo +# MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### choose the engine for mermaid conversion, # default is nodejs, you can change it to playwright,pyppeteer or ink @@ -108,4 +108,4 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### repair operation on the content extracted from LLM's raw output. Warning, it improves the result but not fix all cases. # REPAIR_LLM_OUTPUT: false -PROMPT_FORMAT: json #json or markdown \ No newline at end of file +# PROMPT_FORMAT: json #json or markdown \ No newline at end of file diff --git a/metagpt/config.py b/metagpt/config.py index 8ad42c99f..19bd02c87 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -114,7 +114,7 @@ def _update(self): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.repair_llm_output = self._get("REPAIR_LLM_OUTPUT", False) - self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.prompt_format = self._get("PROMPT_FORMAT", "json") self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) self._ensure_workspace_exists() From 2c68b42432a86f1b1de95bb5e8ede2ba79efcc03 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 12:06:27 +0800 Subject: [PATCH 229/232] action: add example --- metagpt/actions/write_code_review.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 62e96acd8..4b3e9aece 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -66,8 +66,28 @@ 6. ... ## Actions -1. fix class A -2. implement function B +1. Fix the `handle_events` method to update the game state only if a move is successful. + ```python + def handle_events(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + return False + if event.type == pygame.KEYDOWN: + moved = False + if event.key == pygame.K_UP: + moved = self.game.move('UP') + elif event.key == pygame.K_DOWN: + moved = self.game.move('DOWN') + elif event.key == pygame.K_LEFT: + moved = self.game.move('LEFT') + elif event.key == pygame.K_RIGHT: + moved = self.game.move('RIGHT') + if moved: + # Update the game state only if a move was successful + self.render() + return True + ``` +2. Implement function B ## Code Review Result LBTM From bc9f0f190269c23050a4ebf54b3ac6e23af0d68e Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 12:17:26 +0800 Subject: [PATCH 230/232] workspace path update --- metagpt/actions/prepare_documents.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 05255dcc5..8d3445ae4 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -27,8 +27,8 @@ async def run(self, with_messages, **kwargs): # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() workdir = CONFIG.project_path - if not workdir and CONFIG.workspace: - workdir = Path(CONFIG.workspace) / project_name + if not workdir and CONFIG.workspace_path: + workdir = Path(CONFIG.workspace_path) / project_name workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name) if not CONFIG.inc and workdir.exists(): shutil.rmtree(workdir) From a3d7b0f380c8305ce51f0675af74d2e438b7e2b0 Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 13:19:04 +0800 Subject: [PATCH 231/232] CR update --- metagpt/actions/action.py | 1 - metagpt/actions/action_node.py | 3 ++- metagpt/actions/research.py | 2 +- metagpt/actions/summarize_code.py | 14 +++++++------- metagpt/roles/role.py | 16 ---------------- 5 files changed, 10 insertions(+), 26 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 6c1f63f45..1534b1f4d 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -43,7 +43,6 @@ def set_prefix(self, prefix, profile): """Set prefix for later usage""" self.prefix = prefix self.profile = profile - self.llm.system_prompt = prefix return self def __str__(self): diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 1d808ec70..fb7d621d8 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -70,7 +70,8 @@ class ActionNode: content: str instruct_content: BaseModel - def __init__(self, key, expected_type, instruction, example, content="", children=None): + def __init__(self, key: str, expected_type: Type, instruction: str, example: str, content: str = "", + children: dict[str, "ActionNode"] = None): self.key = key self.expected_type = expected_type self.instruction = instruction diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index d7a2a7e38..a70038c51 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -114,7 +114,7 @@ async def run( keywords = OutputParser.extract_struct(keywords, list) keywords = parse_obj_as(list[str], keywords) except Exception as e: - logger.exception(f'fail to get keywords related to the research topic "{topic}" for {e}') + logger.exception(f"fail to get keywords related to the research topic '{topic}' for {e}") keywords = [topic] results = await asyncio.gather(*(self.search_engine.run(i, as_string=False) for i in keywords)) diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index 413ac2a21..f8d8d2b47 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -34,13 +34,13 @@ ----- {code_blocks} -## Code Review All: 请你对历史所有文件进行阅读，在文件中找到可能的bug，如函数未实现、调用错误、未引用等 +## Code Review All: Please read all historical files and find possible bugs in the files, such as unimplemented functions, calling errors, unreferences, etc. -## Call flow: mermaid代码，根据实现的函数，使用mermaid绘制完整的调用链 +## Call flow: mermaid code, based on the implemented function, use mermaid to draw a complete call chain -## Summary: 根据历史文件的实现情况进行总结 +## Summary: Summary based on the implementation of historical files -## TODOs: Python dict[str, str]，这里写出需要修改的文件列表与理由，我们会在之后进行修改 +## TODOs: Python dict[str, str], write down the list of files that need to be modified and the reasons. We will modify them later. """ @@ -49,9 +49,9 @@ ## Code Review All ### a.py -- 它少实现了xxx需求... -- 字段yyy没有给出... -- ... +- It fulfills less of xxx requirements... +- Field yyy is not given... +-... ### b.py ... diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b673c330d..b07541b09 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -218,22 +218,6 @@ def set_env(self, env: "Environment"): if env: env.set_subscription(self, self._subscription) - # # Replaced by FileRepository.set_file - # def set_doc(self, content: str, filename: str): - # return self._rc.env.set_doc(content, filename) - # - # # Replaced by FileRepository.get_file - # def get_doc(self, filename: str): - # return self._rc.env.get_doc(filename) - # - # # Replaced by CONFIG.xx - # def set(self, k, v): - # return self._rc.env.set(k, v) - # - # # Replaced by CONFIG.xx - # def get(self, k): - # return self._rc.env.get(k) - @property def profile(self): """Get the role description (position)""" From df2e9a12be0f9c891405b54fb17c23640d404aae Mon Sep 17 00:00:00 2001 From: geekan Date: Fri, 15 Dec 2023 12:17:26 +0800 Subject: [PATCH 232/232] workspace path update --- metagpt/actions/debug_error.py | 2 +- metagpt/actions/prepare_documents.py | 4 ++-- metagpt/document.py | 1 + metagpt/document_store/document.py | 1 + metagpt/roles/qa_engineer.py | 5 +++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index df60c2e61..39f3bc1bc 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -62,7 +62,7 @@ async def run(self, *args, **kwargs) -> str: if matches: return "" - logger.info(f"Debug and rewrite {self.context.code_filename}") + logger.info(f"Debug and rewrite {self.context.test_filename}") code_doc = await FileRepository.get_file( filename=self.context.code_filename, relative_path=CONFIG.src_workspace ) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 05255dcc5..8d3445ae4 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -27,8 +27,8 @@ async def run(self, with_messages, **kwargs): # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() workdir = CONFIG.project_path - if not workdir and CONFIG.workspace: - workdir = Path(CONFIG.workspace) / project_name + if not workdir and CONFIG.workspace_path: + workdir = Path(CONFIG.workspace_path) / project_name workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name) if not CONFIG.inc and workdir.exists(): shutil.rmtree(workdir) diff --git a/metagpt/document.py b/metagpt/document.py index 6ac4834aa..0af3a915c 100644 --- a/metagpt/document.py +++ b/metagpt/document.py @@ -4,6 +4,7 @@ @Time : 2023/6/8 14:03 @Author : alexanderwu @File : document.py +@Desc : Classes and Operations Related to Files in the File System. """ from enum import Enum from pathlib import Path diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py index c59056312..90abc54de 100644 --- a/metagpt/document_store/document.py +++ b/metagpt/document_store/document.py @@ -4,6 +4,7 @@ @Time : 2023/6/8 14:03 @Author : alexanderwu @File : document.py +@Desc : Classes and Operations Related to Vector Files in the Vector Database. Still under design. """ from pathlib import Path diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index c1573e63b..4439b9b19 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -26,6 +26,7 @@ from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext from metagpt.utils.common import any_to_str_set, parse_recipient +from metagpt.utils.file_repository import FileRepository class QaEngineer(Role): @@ -125,8 +126,8 @@ async def _run_code(self, msg): async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) code = await DebugError(context=run_code_context, llm=self._llm).run() - await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).save( - filename=run_code_context.code_filename, content=code + await FileRepository.save_file( + filename=run_code_context.test_filename, content=code, relative_path=TEST_CODES_FILE_REPO ) run_code_context.output = None self.publish_message(