diff --git a/.gitignore b/.gitignore index e03eab3d3..0ac318ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ cover/ # Django stuff: *.log +logs local_settings.py db.sqlite3 db.sqlite3-journal diff --git a/config/config.yaml b/config/config.yaml index 9acdbe8a1..ef8575e43 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -34,6 +34,15 @@ RPM: 10 #### if zhipuai from `https://open.bigmodel.cn`. You can set here or export API_KEY="YOUR_API_KEY" # ZHIPUAI_API_KEY: "YOUR_API_KEY" +#### if use self-host open llm model with openai-compatible interface +#OPEN_LLM_API_BASE: "http://127.0.0.1:8000/v1" +#OPEN_LLM_API_MODEL: "llama2-13b" +# +##### if use Fireworks api +#FIREWORKS_API_KEY: "YOUR_API_KEY" +#FIREWORKS_API_BASE: "https://api.fireworks.ai/inference/v1" +#FIREWORKS_API_MODEL: "YOUR_LLM_MODEL" # example, accounts/fireworks/models/llama-v2-13b-chat + #### for Search ## Supported values: serpapi/google/serper/ddg @@ -94,4 +103,9 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### browser path for pyppeteer engine, support Chrome, Chromium,MS Edge #PYPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable" +### for repair non-openai LLM's output when parse json-text if PROMPT_FORMAT=json +### due to non-openai LLM's output will not always follow the instruction, so here activate a post-process +### repair operation on the content extracted from LLM's raw output. Warning, it improves the result but not fix all cases. +# REPAIR_LLM_OUTPUT: false + PROMPT_FORMAT: json #json or markdown \ No newline at end of file diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 2fd130cf5..1d9be60e0 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : action.py """ -import re + from abc import ABC from typing import Optional @@ -14,8 +14,9 @@ from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess from metagpt.utils.common import OutputParser -from metagpt.utils.custom_decoder import CustomDecoder +from metagpt.utils.utils import general_after_log class Action(ABC): @@ -57,7 +58,11 @@ async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> s system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) - @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) + @retry( + wait=wait_random_exponential(min=1, max=60), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) async def _aask_v1( self, prompt: str, @@ -67,24 +72,16 @@ async def _aask_v1( format="markdown", # compatible to original format ) -> ActionOutput: content = await self.llm.aask(prompt, system_msgs) - logger.debug(content) + logger.debug(f"llm raw output:\n{content}") output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) if format == "json": - pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" - matches = re.findall(pattern, content, re.DOTALL) - - for match in matches: - if match: - content = match - break - - parsed_data = CustomDecoder(strict=False).decode(content) + parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]") else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) - logger.debug(parsed_data) + logger.debug(f"parsed_data:\n{parsed_data}") instruct_content = output_class(**parsed_data) return ActionOutput(content, instruct_content) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index b1fbdaae9..f5009f345 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -5,13 +5,12 @@ @Author : alexanderwu @File : action_node.py """ -import re -from typing import Dict, Type, List, Any, Tuple, Optional import json +import re +from typing import Any, Dict, List, Optional, Type from pydantic import BaseModel, create_model, root_validator, validator -# , model_validator, field_validator -from tenacity import wait_random_exponential, stop_after_attempt, retry +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions import ActionOutput from metagpt.llm import BaseGPTAPI @@ -53,6 +52,7 @@ def dict_to_markdown(d, prefix="-", postfix="\n"): class ActionNode: """ActionNode is a tree of nodes.""" + # Action Strgy # - sop: 仅使用一级SOP # - complex: 使用一级SOP+自定义策略填槽 @@ -74,8 +74,7 @@ class ActionNode: content: str instruct_content: BaseModel - def __init__(self, key, expected_type, instruction, example, content="", - children=None): + def __init__(self, key, expected_type, instruction, example, content="", children=None): self.key = key self.expected_type = expected_type self.instruction = instruction @@ -84,8 +83,9 @@ def __init__(self, key, expected_type, instruction, example, content="", self.children = children if children is not None else {} def __str__(self): - return f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" \ - f", {self.content}, {self.children}" + return ( + f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" f", {self.content}, {self.children}" + ) def __repr__(self): return self.__str__() @@ -116,7 +116,7 @@ def get_self_mapping(self) -> Dict[str, Type]: def get_mapping(self, mode="children") -> Dict[str, Type]: """get key: type mapping under mode""" - if mode == "children" or (mode=="auto" and self.children): + if mode == "children" or (mode == "auto" and self.children): return self.get_children_mapping() return self.get_self_mapping() @@ -148,7 +148,7 @@ def create_model_class_v2(cls, class_name: str, mapping: Dict[str, Type]): """基于pydantic v2的模型动态生成，用来检验结果类型正确性，待验证""" new_class = create_model(class_name, **mapping) - @model_validator(mode='before') + @model_validator(mode="before") def check_missing_fields(data): required_fields = set(mapping.keys()) missing_fields = required_fields - set(data.keys()) @@ -156,7 +156,7 @@ def check_missing_fields(data): raise ValueError(f"Missing fields: {missing_fields}") return data - @field_validator('*') + @field_validator("*") def check_name(v: Any, field: str) -> Any: if field not in mapping.keys(): raise ValueError(f"Unrecognized block: {field}") @@ -242,8 +242,9 @@ def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) # FIXME: json instruction会带来格式问题，如："Project name": "web_2048 # 项目名称使用下划线", self.instruction = self.compile_instruction(to="markdown", mode=mode) self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) - prompt = template.format(context=context, example=self.example, instruction=self.instruction, - constraint=CONSTRAINT) + prompt = template.format( + context=context, example=self.example, instruction=self.instruction, constraint=CONSTRAINT + ) return prompt @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) @@ -302,7 +303,7 @@ async def simple_fill(self, to, mode): return self async def fill(self, context, llm, to="json", mode="auto", strgy="simple"): - """ Fill the node(s) with mode. + """Fill the node(s) with mode. :param context: Everything we should know when filling node. :param llm: Large Language Model with pre-defined system message. @@ -336,9 +337,7 @@ async def fill(self, context, llm, to="json", mode="auto", strgy="simple"): def action_node_from_tuple_example(): # 示例：列表中包含元组 - list_of_tuples = [ - ("key1", str, "Instruction 1", "Example 1") - ] + list_of_tuples = [("key1", str, "Instruction 1", "Example 1")] # 从列表中创建 ActionNode 实例 nodes = [ActionNode(*data) for data in list_of_tuples] @@ -346,5 +345,5 @@ def action_node_from_tuple_example(): logger.info(i) -if __name__ == '__main__': +if __name__ == "__main__": action_node_from_tuple_example() diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index e66eadeaa..df60c2e61 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -42,7 +42,7 @@ ``` --- Now you should start rewriting the code: -## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. +## file name of the code to rewrite: Write code with triple quote. Do your best to implement THIS IN ONLY ONE FILE. """ diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index fd58e0ca8..5a5f52de7 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,6 @@ """ import json from pathlib import Path -# from typing import List from metagpt.actions import Action, ActionOutput from metagpt.actions.design_api_an import DESIGN_API_NODE @@ -26,9 +25,13 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository + # from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file +# from typing import List + + NEW_REQ_TEMPLATE = """ ### Legacy Content {old_design} @@ -82,9 +85,7 @@ async def _new_system_design(self, context, format=CONFIG.prompt_format): return node async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - context = NEW_REQ_TEMPLATE.format( - old_design=system_design_doc.content, context=prd_doc.content - ) + context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content) node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format) system_design_doc.content = node.instruct_content.json(ensure_ascii=False) return system_design_doc diff --git a/metagpt/actions/design_api_an.py b/metagpt/actions/design_api_an.py index 2db203606..0a303cdd5 100644 --- a/metagpt/actions/design_api_an.py +++ b/metagpt/actions/design_api_an.py @@ -6,52 +6,49 @@ @File : design_api_an.py """ from metagpt.actions.action_node import ActionNode -from metagpt.utils.mermaid import MMC1, MMC2 from metagpt.logs import logger +from metagpt.utils.mermaid import MMC1, MMC2 IMPLEMENTATION_APPROACH = ActionNode( key="Implementation approach", expected_type=str, instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework", - example="We will ..." + example="We will ...", ) PROJECT_NAME = ActionNode( - key="Project name", - expected_type=str, - instruction="The project name with underline", - example="game_2048" + key="Project name", expected_type=str, instruction="The project name with underline", example="game_2048" ) FILE_LIST = ActionNode( key="File list", expected_type=list[str], instruction="Only need relative paths. ALWAYS write a main.py or app.py here", - example=['main.py', 'game.py'] + example=["main.py", "game.py"], ) DATA_STRUCTURES_AND_INTERFACES = ActionNode( key="Data structures and interfaces", expected_type=str, instruction="Use mermaid classDiagram code syntax, including classes, method(__init__ etc.) and functions with type" - " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " - "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", - example=MMC1 + " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. " + "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.", + example=MMC1, ) PROGRAM_CALL_FLOW = ActionNode( key="Program call flow", expected_type=str, instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE " - "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", - example=MMC2 + "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.", + example=MMC2, ) ANYTHING_UNCLEAR = ActionNode( key="Anything UNCLEAR", expected_type=str, instruction="Mention unclear project aspects, then try to clarify it.", - example="Clarification needed on third-party API integration, ..." + example="Clarification needed on third-party API integration, ...", ) NODES = [ @@ -60,7 +57,7 @@ FILE_LIST, DATA_STRUCTURES_AND_INTERFACES, PROGRAM_CALL_FLOW, - ANYTHING_UNCLEAR + ANYTHING_UNCLEAR, ] DESIGN_API_NODE = ActionNode.from_children("DesignAPI", NODES) @@ -71,5 +68,5 @@ def main(): logger.info(prompt) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 29e3bed3e..c95be4012 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -10,7 +10,6 @@ 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ import json -# from typing import List from metagpt.actions import ActionOutput from metagpt.actions.action import Action @@ -25,6 +24,9 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository + +# from typing import List + # from metagpt.utils.get_template import get_template NEW_REQ_TEMPLATE = """ @@ -97,7 +99,8 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format): async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content) node = await PM_NODE.fill(context, self.llm, format) - return node + task_doc.content = node.content + return task_doc @staticmethod async def _update_requirements(doc): diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py index 9849cb7b3..e03af36d7 100644 --- a/metagpt/actions/project_management_an.py +++ b/metagpt/actions/project_management_an.py @@ -12,51 +12,53 @@ key="Required Python packages", expected_type=list[str], instruction="Provide required Python packages in requirements.txt format.", - example=["flask==1.1.2", "bcrypt==3.2.0"] + example=["flask==1.1.2", "bcrypt==3.2.0"], ) REQUIRED_OTHER_LANGUAGE_PACKAGES = ActionNode( key="Required Other language third-party packages", expected_type=list[str], instruction="List down the required packages for languages other than Python.", - example=["No third-party dependencies required"] + example=["No third-party dependencies required"], ) LOGIC_ANALYSIS = ActionNode( key="Logic Analysis", expected_type=list[list[str]], instruction="Provide a list of files with the classes/methods/functions to be implemented, " - "including dependency analysis and imports.", - example=[["game.py", "Contains Game class and ... functions"], - ["main.py", "Contains main function, depends on game.py"]] + "including dependency analysis and imports.", + example=[ + ["game.py", "Contains Game class and ... functions"], + ["main.py", "Contains main function, from game import Game"], + ], ) TASK_LIST = ActionNode( key="Task list", expected_type=list[str], instruction="Break down the tasks into a list of filenames, prioritized by dependency order.", - example=["game.py", "main.py"] + example=["game.py", "main.py"], ) FULL_API_SPEC = ActionNode( key="Full API spec", expected_type=str, instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend.", - example="openapi: 3.0.0 ..." + example="openapi: 3.0.0 ...", ) SHARED_KNOWLEDGE = ActionNode( key="Shared Knowledge", expected_type=str, instruction="Detail any shared knowledge, like common utility functions or configuration variables.", - example="'game.py' contains functions shared across the project." + example="'game.py' contains functions shared across the project.", ) ANYTHING_UNCLEAR_PM = ActionNode( key="Anything UNCLEAR", expected_type=str, instruction="Mention any unclear aspects in the project management context and try to clarify them.", - example="Clarification needed on how to start and initialize third-party libraries." + example="Clarification needed on how to start and initialize third-party libraries.", ) NODES = [ @@ -66,7 +68,7 @@ TASK_LIST, FULL_API_SPEC, SHARED_KNOWLEDGE, - ANYTHING_UNCLEAR_PM + ANYTHING_UNCLEAR_PM, ] @@ -78,5 +80,5 @@ def main(): logger.info(prompt) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 1dda6466f..a2501db2a 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -20,8 +20,13 @@ from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO, BUGFIX_FILENAME, \ - DOCS_FILE_REPO +from metagpt.const import ( + BUGFIX_FILENAME, + CODE_SUMMARIES_FILE_REPO, + DOCS_FILE_REPO, + TASK_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, +) from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser @@ -143,4 +148,3 @@ async def _get_codes(task_doc) -> str: continue codes.append(doc.content) return "\n----------\n".join(codes) - diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index e61743e7f..bb0cf8fb9 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -14,28 +14,35 @@ import json from pathlib import Path -# from typing import List from metagpt.actions import Action, ActionOutput from metagpt.actions.action_node import ActionNode -from metagpt.actions.write_prd_an import WRITE_PRD_NODE, WP_ISSUE_TYPE_NODE, WP_IS_RELATIVE_NODE from metagpt.actions.fix_bug import FixBug -from metagpt.actions.search_and_summarize import SearchAndSummarize +from metagpt.actions.write_prd_an import ( + WP_IS_RELATIVE_NODE, + WP_ISSUE_TYPE_NODE, + WRITE_PRD_NODE, +) from metagpt.config import CONFIG from metagpt.const import ( + BUGFIX_FILENAME, COMPETITIVE_ANALYSIS_FILE_REPO, DOCS_FILE_REPO, PRD_PDF_FILE_REPO, PRDS_FILE_REPO, - REQUIREMENT_FILENAME, BUGFIX_FILENAME, + REQUIREMENT_FILENAME, ) from metagpt.logs import logger -from metagpt.schema import Document, Documents, Message, BugFixContext +from metagpt.schema import BugFixContext, Document, Documents, Message from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository + # from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file +# from typing import List + + CONTEXT_TEMPLATE = """ ### Project Name {project_name} @@ -69,12 +76,14 @@ async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") bug_fix = BugFixContext(filename=BUGFIX_FILENAME) - return Message(content=bug_fix.json(), instruct_content=bug_fix, - role="", - cause_by=FixBug, - sent_from=self, - send_to="Alex", # the name of Engineer - ) + return Message( + content=bug_fix.json(), + instruct_content=bug_fix, + role="", + cause_by=FixBug, + sent_from=self, + send_to="Alex", # the name of Engineer + ) else: await docs_file_repo.delete(filename=BUGFIX_FILENAME) @@ -154,7 +163,7 @@ async def _save_competitive_analysis(prd_doc): if not quadrant_chart: return pathname = ( - CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") ) if not pathname.parent.exists(): pathname.parent.mkdir(parents=True, exist_ok=True) @@ -181,5 +190,9 @@ async def _rename_workspace(prd): CONFIG.git_repo.rename_root(CONFIG.project_name) async def _is_bugfix(self, context) -> bool: + src_workspace_path = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name + code_files = CONFIG.git_repo.get_files(relative_path=src_workspace_path) + if not code_files: + return False node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm) return node.get("issue_type") == "BUG" diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index cbcf920b9..2c81bdb6e 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -13,52 +13,52 @@ key="Language", expected_type=str, instruction="Provide the language used in the project, typically matching the user's requirement language.", - example="en_us" + example="en_us", ) PROGRAMMING_LANGUAGE = ActionNode( key="Programming Language", expected_type=str, instruction="Python/JavaScript or other mainstream programming language.", - example="Python" + example="Python", ) ORIGINAL_REQUIREMENTS = ActionNode( key="Original Requirements", expected_type=str, instruction="Place the polished, complete original requirements here.", - example="The game should have a leaderboard and multiple difficulty levels." + example="The game should have a leaderboard and multiple difficulty levels.", ) PROJECT_NAME = ActionNode( key="Project Name", expected_type=str, instruction="Name the project using snake case style, like 'game_2048' or 'simple_crm'.", - example="game_2048" + example="game_2048", ) PRODUCT_GOALS = ActionNode( key="Product Goals", expected_type=list[str], instruction="Provide up to three clear, orthogonal product goals.", - example=["Create an engaging user experience", - "Ensure high performance", - "Provide customizable features"] + example=["Create an engaging user experience", "Ensure high performance", "Provide customizable features"], ) USER_STORIES = ActionNode( key="User Stories", expected_type=list[str], instruction="Provide up to five scenario-based user stories.", - example=["As a user, I want to be able to choose difficulty levels", - "As a player, I want to see my score after each game"] + example=[ + "As a user, I want to be able to choose difficulty levels", + "As a player, I want to see my score after each game", + ], ) COMPETITIVE_ANALYSIS = ActionNode( key="Competitive Analysis", expected_type=list[str], instruction="Provide analyses for up to seven competitive products.", - example=["Python Snake Game: Simple interface, lacks advanced features"] + example=["Python Snake Game: Simple interface, lacks advanced features"], ) COMPETITIVE_QUADRANT_CHART = ActionNode( @@ -79,56 +79,53 @@ "Campaign D": [0.78, 0.34] "Campaign E": [0.40, 0.34] "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6]""" + "Our Target Product": [0.5, 0.6]""", ) REQUIREMENT_ANALYSIS = ActionNode( key="Requirement Analysis", expected_type=str, instruction="Provide a detailed analysis of the requirements.", - example="The product should be user-friendly and performance-optimized." + example="The product should be user-friendly and performance-optimized.", ) REQUIREMENT_POOL = ActionNode( key="Requirement Pool", expected_type=list[list[str]], instruction="List down the requirements with their priority (P0, P1, P2).", - example=[["P0", "High priority requirement"], ["P1", "Medium priority requirement"]] + example=[["P0", "High priority requirement"], ["P1", "Medium priority requirement"]], ) UI_DESIGN_DRAFT = ActionNode( key="UI Design draft", expected_type=str, instruction="Provide a simple description of UI elements, functions, style, and layout.", - example="Basic function description with a simple style and layout." + example="Basic function description with a simple style and layout.", ) ANYTHING_UNCLEAR = ActionNode( key="Anything UNCLEAR", expected_type=str, instruction="Mention any aspects of the project that are unclear and try to clarify them.", - example="..." + example="...", ) ISSUE_TYPE = ActionNode( key="issue_type", expected_type=str, instruction="Answer BUG/REQUIREMENT. If it is a bugfix, answer BUG, otherwise answer Requirement", - example="BUG" + example="BUG", ) IS_RELATIVE = ActionNode( key="is_relative", expected_type=str, instruction="Answer YES/NO. If the requirement is related to the old PRD, answer YES, otherwise NO", - example="YES" + example="YES", ) REASON = ActionNode( - key="reason", - expected_type=str, - instruction="Explain the reasoning process from question to answer", - example="..." + key="reason", expected_type=str, instruction="Explain the reasoning process from question to answer", example="..." ) @@ -144,7 +141,7 @@ REQUIREMENT_ANALYSIS, REQUIREMENT_POOL, UI_DESIGN_DRAFT, - ANYTHING_UNCLEAR + ANYTHING_UNCLEAR, ] WRITE_PRD_NODE = ActionNode.from_children("WritePRD", NODES) @@ -157,5 +154,5 @@ def main(): logger.info(prompt) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 65673807f..9dd967788 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -31,7 +31,7 @@ ``` Note that the code to test is at {source_file_path}, we will put your test code at {workspace}/tests/{test_file_name}, and run your test code from {workspace}, you should correctly import the necessary classes based on these file locations! -## {test_file_name}: Write test code with triple quoto. Do your best to implement THIS ONLY ONE FILE. +## {test_file_name}: Write test code with triple quote. Do your best to implement THIS ONLY ONE FILE. """ diff --git a/metagpt/config.py b/metagpt/config.py index d2390f704..d04da1d91 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -55,12 +55,20 @@ def _update(self): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY") + self.open_llm_api_base = self._get("OPEN_LLM_API_BASE") + self.open_llm_api_model = self._get("OPEN_LLM_API_MODEL") + self.fireworks_api_key = self._get("FIREWORKS_API_KEY") if ( (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key) + and (not self.open_llm_api_base) + and (not self.fireworks_api_key or "YOUR_API_KEY" == self.fireworks_api_key) ): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") + raise NotConfiguredException( + "Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first " + "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE" + ) self.openai_api_base = self._get("OPENAI_API_BASE") self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") @@ -77,6 +85,9 @@ def _update(self): self.domain = self._get("DOMAIN") self.spark_url = self._get("SPARK_URL") + self.fireworks_api_base = self._get("FIREWORKS_API_BASE") + self.fireworks_api_model = self._get("FIREWORKS_API_MODEL") + self.claude_api_key = self._get("Anthropic_API_KEY") self.serpapi_api_key = self._get("SERPAPI_API_KEY") self.serper_api_key = self._get("SERPER_API_KEY") @@ -102,6 +113,7 @@ def _update(self): self.mermaid_engine = self._get("MERMAID_ENGINE", "nodejs") self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") + self.repair_llm_output = self._get("REPAIR_LLM_OUTPUT", False) self.prompt_format = self._get("PROMPT_FORMAT", "markdown") self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) self._ensure_workspace_exists() diff --git a/metagpt/llm.py b/metagpt/llm.py index eaa4880a5..7c0ad7975 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -7,12 +7,13 @@ """ from metagpt.config import CONFIG -from metagpt.provider.anthropic_api import Claude2 as Claude +from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.provider.fireworks_api import FireWorksGPTAPI from metagpt.provider.human_provider import HumanProvider +from metagpt.provider.open_llm_api import OpenLLMGPTAPI from metagpt.provider.openai_api import OpenAIGPTAPI from metagpt.provider.spark_api import SparkAPI from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI -from metagpt.provider.base_gpt_api import BaseGPTAPI _ = HumanProvider() # Avoid pre-commit error @@ -22,12 +23,14 @@ def LLM() -> BaseGPTAPI: # TODO a little trick, can use registry to initialize LLM instance further if CONFIG.openai_api_key: llm = OpenAIGPTAPI() - elif CONFIG.claude_api_key: - llm = Claude() elif CONFIG.spark_api_key: llm = SparkAPI() elif CONFIG.zhipuai_api_key: llm = ZhiPuAIGPTAPI() + elif CONFIG.open_llm_api_base: + llm = OpenLLMGPTAPI() + elif CONFIG.fireworks_api_key: + llm = FireWorksGPTAPI() else: raise RuntimeError("You should config a LLM configuration first") diff --git a/metagpt/logs.py b/metagpt/logs.py index 092b4c476..ab1bc4e94 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -7,6 +7,7 @@ """ import sys +from datetime import datetime from loguru import logger as _logger @@ -15,9 +16,12 @@ def define_log_level(print_level="INFO", logfile_level="DEBUG"): """Adjust the log level to above level""" + current_date = datetime.now() + formatted_date = current_date.strftime("%Y%m%d") + _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(METAGPT_ROOT / "logs/log.txt", level=logfile_level) + _logger.add(METAGPT_ROOT / f"logs/{formatted_date}.txt", level=logfile_level) return _logger diff --git a/metagpt/prompts/generate_skill.md b/metagpt/prompts/generate_skill.md index 74948cd15..e96f8181a 100644 --- a/metagpt/prompts/generate_skill.md +++ b/metagpt/prompts/generate_skill.md @@ -10,7 +10,7 @@ For instance, if the context is: from typing import Optional from abc import ABC from metagpt.llm import LLM # Large language model, similar to GPT -n + class Action(ABC): def __init__(self, name='', context=None, llm: LLM = LLM()): self.name = name diff --git a/metagpt/provider/fireworks_api.py b/metagpt/provider/fireworks_api.py new file mode 100644 index 000000000..47ac9cf61 --- /dev/null +++ b/metagpt/provider/fireworks_api.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : fireworks.ai's api + +import openai + +from metagpt.config import CONFIG +from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter + + +class FireWorksGPTAPI(OpenAIGPTAPI): + def __init__(self): + self.__init_fireworks(CONFIG) + self.llm = openai + self.model = CONFIG.fireworks_api_model + self.auto_max_tokens = False + self._cost_manager = CostManager() + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_fireworks(self, config: "Config"): + openai.api_key = config.fireworks_api_key + openai.api_base = config.fireworks_api_base + self.rpm = int(config.get("RPM", 10)) diff --git a/metagpt/provider/open_llm_api.py b/metagpt/provider/open_llm_api.py new file mode 100644 index 000000000..f421e30c8 --- /dev/null +++ b/metagpt/provider/open_llm_api.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : self-host open llm model with openai-compatible interface + +import openai + +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter + + +class OpenLLMCostManager(CostManager): + """open llm model is self-host, it's free and without cost""" + + def update_cost(self, prompt_tokens, completion_tokens, model): + """ + Update the total cost, prompt tokens, and completion tokens. + + Args: + prompt_tokens (int): The number of tokens used in the prompt. + completion_tokens (int): The number of tokens used in the completion. + model (str): The model used for the API call. + """ + self.total_prompt_tokens += prompt_tokens + self.total_completion_tokens += completion_tokens + + logger.info( + f"Max budget: ${CONFIG.max_budget:.3f} | " + f"prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" + ) + CONFIG.total_cost = self.total_cost + + +class OpenLLMGPTAPI(OpenAIGPTAPI): + def __init__(self): + self.__init_openllm(CONFIG) + self.llm = openai + self.model = CONFIG.open_llm_api_model + self.auto_max_tokens = False + self._cost_manager = OpenLLMCostManager() + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_openllm(self, config: "Config"): + openai.api_key = "sk-xx" # self-host api doesn't need api-key, use the default value + openai.api_base = config.open_llm_api_base + self.rpm = int(config.get("RPM", 10)) diff --git a/metagpt/provider/postprecess/__init__.py b/metagpt/provider/postprecess/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/metagpt/provider/postprecess/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/metagpt/provider/postprecess/base_postprecess_plugin.py b/metagpt/provider/postprecess/base_postprecess_plugin.py new file mode 100644 index 000000000..0d1cfbb11 --- /dev/null +++ b/metagpt/provider/postprecess/base_postprecess_plugin.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : base llm postprocess plugin to do the operations like repair the raw llm output + +from typing import Union + +from metagpt.logs import logger +from metagpt.utils.repair_llm_raw_output import ( + RepairType, + extract_content_from_output, + repair_llm_raw_output, + retry_parse_json_text, +) + + +class BasePostPrecessPlugin(object): + model = None # the plugin of the `model`, use to judge in `llm_postprecess` + + def run_repair_llm_output(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]: + """ + repair steps + 1. repair the case sensitive problem using the schema's fields + 2. extract the content from the req_key pair( xx[REQ_KEY]xxx[/REQ_KEY]xx ) + 3. repair the invalid json text in the content + 4. parse the json text and repair it according to the exception with retry loop + """ + output_class_fields = list(schema["properties"].keys()) # Custom ActionOutput's fields + + content = self.run_repair_llm_raw_output(output, req_keys=output_class_fields + [req_key]) + content = self.run_extract_content_from_output(content, right_key=req_key) + # # req_keys mocked + content = self.run_repair_llm_raw_output(content, req_keys=[None], repair_type=RepairType.JSON) + parsed_data = self.run_retry_parse_json_text(content) + + return parsed_data + + def run_repair_llm_raw_output(self, content: str, req_keys: list[str], repair_type: str = None) -> str: + """inherited class can re-implement the function""" + return repair_llm_raw_output(content, req_keys=req_keys, repair_type=repair_type) + + def run_extract_content_from_output(self, content: str, right_key: str) -> str: + """inherited class can re-implement the function""" + return extract_content_from_output(content, right_key=right_key) + + def run_retry_parse_json_text(self, content: str) -> Union[dict, list]: + """inherited class can re-implement the function""" + logger.info(f"extracted json CONTENT from output:\n{content}") + parsed_data = retry_parse_json_text(output=content) # should use output=content + return parsed_data + + def run(self, output: str, schema: dict, req_key: str = "[/CONTENT]") -> Union[dict, list]: + """ + this is used for prompt with a json-format output requirement and outer pair key, like + [REQ_KEY] + { + "Key": "value" + } + [/REQ_KEY] + + Args + outer (str): llm raw output + schema: output json schema + req_key: outer pair right key, usually in `[/REQ_KEY]` format + """ + assert len(schema.get("properties")) > 0 + assert "/" in req_key + + # current, postprocess only deal the repair_llm_raw_output + new_output = self.run_repair_llm_output(output=output, schema=schema, req_key=req_key) + return new_output diff --git a/metagpt/provider/postprecess/llm_output_postprecess.py b/metagpt/provider/postprecess/llm_output_postprecess.py new file mode 100644 index 000000000..85405543d --- /dev/null +++ b/metagpt/provider/postprecess/llm_output_postprecess.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : the entry of choosing which PostProcessPlugin to deal particular LLM model's output + +from typing import Union + +from metagpt.provider.postprecess.base_postprecess_plugin import BasePostPrecessPlugin + + +def llm_output_postprecess( + output: str, schema: dict, req_key: str = "[/CONTENT]", model_name: str = None +) -> Union[dict, str]: + """ + default use BasePostPrecessPlugin if there is not matched plugin. + """ + # TODO choose different model's plugin according to the model_name + postprecess_plugin = BasePostPrecessPlugin() + + result = postprecess_plugin.run(output=output, schema=schema, req_key=req_key) + return result diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index b80ef85be..2c0bdd1d6 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -27,7 +27,7 @@ def __init__( name: str = "Bob", profile: str = "Architect", goal: str = "design a concise, usable, complete software system", - constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries" + constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries", ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 37090b24f..bfe1be251 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -26,7 +26,7 @@ def __init__( name: str = "Eve", profile: str = "Project Manager", goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task " - "dependencies to start with the prerequisite modules", + "dependencies to start with the prerequisite modules", constraints: str = "", ) -> None: """ diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 15a01b9e9..c1573e63b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -14,9 +14,7 @@ @Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results of SummarizeCode. """ -from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest - -# from metagpt.const import WORKSPACE_ROOT +from metagpt.actions import DebugError, RunCode, WriteTest from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 7c9341adb..633ad6051 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -27,14 +27,12 @@ from metagpt.actions import Action, ActionOutput from metagpt.actions.action_node import ActionNode -from metagpt.config import CONFIG from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory - -# from metagpt.memory import LongTermMemory from metagpt.schema import Message, MessageQueue from metagpt.utils.common import any_to_str +from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -113,9 +111,10 @@ class Config: arbitrary_types_allowed = True def check(self, role_id: str): - if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: - self.long_term_memory.recover_memory(role_id, self) - self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation + # if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: + # self.long_term_memory.recover_memory(role_id, self) + # self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation + pass @property def important_memory(self) -> list[Message]: @@ -153,8 +152,9 @@ def _init_actions(self, actions): else: if self._setting.is_human and not isinstance(action.llm, HumanProvider): logger.warning( - f"is_human attribute does not take effect," - f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances" + f"is_human attribute does not take effect, " + f"as Role's {str(action)} was initialized using LLM, " + f"try passing in Action classes instead of initialized instances" ) i = action # i.set_env(self._rc.env) @@ -266,6 +266,7 @@ async def _think(self) -> None: ) # print(prompt) next_state = await self._llm.aask(prompt) + next_state = extract_state_value_from_output(next_state) logger.debug(f"{prompt=}") if (not next_state.isdigit() and next_state != "-1") or int(next_state) not in range(-1, len(self._states)): logger.warning(f"Invalid answer of state, {next_state=}, will be set to -1") diff --git a/metagpt/schema.py b/metagpt/schema.py index 25281e399..baed5582b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -97,14 +97,14 @@ class Message(BaseModel): send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) def __init__( - self, - content, - instruct_content=None, - role="user", - cause_by="", - sent_from="", - send_to=MESSAGE_ROUTE_TO_ALL, - **kwargs, + self, + content, + instruct_content=None, + role="user", + cause_by="", + sent_from="", + send_to=MESSAGE_ROUTE_TO_ALL, + **kwargs, ): """ Parameters not listed below will be stored as meta info, including custom parameters. diff --git a/metagpt/utils/ahttp_client.py b/metagpt/utils/ahttp_client.py new file mode 100644 index 000000000..b4a33e9d7 --- /dev/null +++ b/metagpt/utils/ahttp_client.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : pure async http_client + +from typing import Any, Mapping, Optional, Union + +import aiohttp +from aiohttp.client import DEFAULT_TIMEOUT + + +async def apost( + url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + as_json: bool = False, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total, +) -> Union[str, dict]: + async with aiohttp.ClientSession() as session: + async with session.post(url=url, params=params, json=json, data=data, headers=headers, timeout=timeout) as resp: + if as_json: + data = await resp.json() + else: + data = await resp.read() + data = data.decode(encoding) + return data + + +async def apost_stream( + url: str, + params: Optional[Mapping[str, str]] = None, + json: Any = None, + data: Any = None, + headers: Optional[dict] = None, + encoding: str = "utf-8", + timeout: int = DEFAULT_TIMEOUT.total, +) -> Any: + """ + usage: + result = astream(url="xx") + async for line in result: + deal_with(line) + """ + async with aiohttp.ClientSession() as session: + async with session.post(url=url, params=params, json=json, data=data, headers=headers, timeout=timeout) as resp: + async for line in resp.content: + yield line.decode(encoding) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 9827b8252..1340b1768 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,13 +8,15 @@ """ from __future__ import annotations -from gitignore_parser import parse_gitignore, rule_from_pattern, handle_negation import shutil from enum import Enum from pathlib import Path from typing import Dict, List + from git.repo import Repo from git.repo.fun import is_git_dir +from gitignore_parser import parse_gitignore + from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile @@ -236,8 +238,9 @@ def get_files(self, relative_path: Path | str, root_relative_path: Path | str = rpath = file_path.relative_to(root_relative_path) files.append(str(rpath)) else: - subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path, - filter_ignored=False) + subfolder_files = self.get_files( + relative_path=file_path, root_relative_path=root_relative_path, filter_ignored=False + ) files.extend(subfolder_files) except Exception as e: logger.error(f"Error: {e}") diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py new file mode 100644 index 000000000..4aafd8e66 --- /dev/null +++ b/metagpt/utils/repair_llm_raw_output.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : repair llm raw output with particular conditions + +import copy +from enum import Enum +from typing import Callable, Union + +import regex as re +from tenacity import RetryCallState, retry, stop_after_attempt, wait_fixed + +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.utils.custom_decoder import CustomDecoder + + +class RepairType(Enum): + CS = "case sensitivity" + RKPM = "required key pair missing" # condition like `[key] xx` which lacks `[/key]` + SCM = "special character missing" # Usually the req_key appear in pairs like `[key] xx [/key]` + JSON = "json format" + + +def repair_case_sensitivity(output: str, req_key: str) -> str: + """ + usually, req_key is the key name of expected json or markdown content, it won't appear in the value part. + fix target string `"Shared Knowledge": ""` but `"Shared knowledge": ""` actually + """ + if req_key in output: + return output + + output_lower = output.lower() + req_key_lower = req_key.lower() + if req_key_lower in output_lower: + # find the sub-part index, and replace it with raw req_key + lidx = output_lower.find(req_key_lower) + source = output[lidx : lidx + len(req_key_lower)] + output = output.replace(source, req_key) + logger.info(f"repair_case_sensitivity: {req_key}") + + return output + + +def repair_special_character_missing(output: str, req_key: str = "[/CONTENT]") -> str: + """ + fix + 1. target string `[CONTENT] xx [CONTENT] xxx [CONTENT]` lacks `/` in the last `[CONTENT]` + 2. target string `xx [CONTENT] xxx [CONTENT] xxxx` lacks `/` in the last `[CONTENT]` + """ + sc_arr = ["/"] + + if req_key in output: + return output + + for sc in sc_arr: + req_key_pure = req_key.replace(sc, "") + appear_cnt = output.count(req_key_pure) + if req_key_pure in output and appear_cnt > 1: + # req_key with special_character usually in the tail side + ridx = output.rfind(req_key_pure) + output = f"{output[:ridx]}{req_key}{output[ridx + len(req_key_pure):]}" + logger.info(f"repair_special_character_missing: {sc} in {req_key_pure} as position {ridx}") + + return output + + +def repair_required_key_pair_missing(output: str, req_key: str = "[/CONTENT]") -> str: + """ + implement the req_key pair in the begin or end of the content + req_key format + 1. `[req_key]`, and its pair `[/req_key]` + 2. `[/req_key]`, and its pair `[req_key]` + """ + sc = "/" # special char + if req_key.startswith("[") and req_key.endswith("]"): + if sc in req_key: + left_key = req_key.replace(sc, "") # `[/req_key]` -> `[req_key]` + right_key = req_key + else: + left_key = req_key + right_key = f"{req_key[0]}{sc}{req_key[1:]}" # `[req_key]` -> `[/req_key]` + + if left_key not in output: + output = left_key + "\n" + output + if right_key not in output: + + def judge_potential_json(routput: str, left_key: str) -> Union[str, None]: + ridx = routput.rfind(left_key) + if ridx < 0: + return None + sub_output = routput[ridx:] + idx1 = sub_output.rfind("}") + idx2 = sub_output.rindex("]") + idx = idx1 if idx1 >= idx2 else idx2 + sub_output = sub_output[: idx + 1] + return sub_output + + if output.strip().endswith("}") or (output.strip().endswith("]") and not output.strip().endswith(left_key)): + # # avoid [req_key]xx[req_key] case to append [/req_key] + output = output + "\n" + right_key + elif judge_potential_json(output, left_key) and (not output.strip().endswith(left_key)): + sub_content = judge_potential_json(output, left_key) + output = sub_content + "\n" + right_key + + return output + + +def repair_json_format(output: str) -> str: + """ + fix extra `[` or `}` in the end + """ + output = output.strip() + + if output.startswith("[{"): + output = output[1:] + logger.info(f"repair_json_format: {'[{'}") + elif output.endswith("}]"): + output = output[:-1] + logger.info(f"repair_json_format: {'}]'}") + elif output.startswith("{") and output.endswith("]"): + output = output[:-1] + "}" + + return output + + +def _repair_llm_raw_output(output: str, req_key: str, repair_type: RepairType = None) -> str: + repair_types = [repair_type] if repair_type else [item for item in RepairType if item not in [RepairType.JSON]] + for repair_type in repair_types: + if repair_type == RepairType.CS: + output = repair_case_sensitivity(output, req_key) + elif repair_type == RepairType.RKPM: + output = repair_required_key_pair_missing(output, req_key) + elif repair_type == RepairType.SCM: + output = repair_special_character_missing(output, req_key) + elif repair_type == RepairType.JSON: + output = repair_json_format(output) + return output + + +def repair_llm_raw_output(output: str, req_keys: list[str], repair_type: RepairType = None) -> str: + """ + in open-source llm model, it usually can't follow the instruction well, the output may be incomplete, + so here we try to repair it and use all repair methods by default. + typical case + 1. case sensitivity + target: "Original Requirements" + output: "Original requirements" + 2. special character missing + target: [/CONTENT] + output: [CONTENT] + 3. json format + target: { xxx } + output: { xxx }] + """ + if not CONFIG.repair_llm_output: + return output + + # do the repairation usually for non-openai models + for req_key in req_keys: + output = _repair_llm_raw_output(output=output, req_key=req_key, repair_type=repair_type) + return output + + +def repair_invalid_json(output: str, error: str) -> str: + """ + repair the situation like there are extra chars like + error examples + example 1. json.decoder.JSONDecodeError: Expecting ',' delimiter: line 154 column 1 (char 2765) + example 2. xxx.JSONDecodeError: Expecting property name enclosed in double quotes: line 14 column 1 (char 266) + """ + pattern = r"line ([0-9]+)" + + matches = re.findall(pattern, error, re.DOTALL) + if len(matches) > 0: + line_no = int(matches[0]) - 1 + + # due to CustomDecoder can handle `"": ''` or `'': ""`, so convert `"""` -> `"`, `'''` -> `'` + output = output.replace('"""', '"').replace("'''", '"') + arr = output.split("\n") + line = arr[line_no].strip() + # different general problems + if line.endswith("],"): + # problem, redundant char `]` + new_line = line.replace("]", "") + elif line.endswith("},") and not output.endswith("},"): + # problem, redundant char `}` + new_line = line.replace("}", "") + elif line.endswith("},") and output.endswith("},"): + new_line = line[:-1] + elif '",' not in line and "," not in line: + new_line = f'{line}",' + elif "," not in line: + # problem, miss char `,` at the end. + new_line = f"{line}," + elif "," in line and len(line) == 1: + new_line = f'"{line}' + elif '",' in line: + new_line = line[:-2] + "'," + + arr[line_no] = new_line + output = "\n".join(arr) + logger.info(f"repair_invalid_json, raw error: {error}") + + return output + + +def run_after_exp_and_passon_next_retry(logger: "loguru.Logger") -> Callable[["RetryCallState"], None]: + def run_and_passon(retry_state: RetryCallState) -> None: + """ + RetryCallState example + { + "start_time":143.098322024, + "retry_object":")>", + "fn":"", + "args":"(\"tag:[/CONTENT]\",)", # function input args + "kwargs":{}, # function input kwargs + "attempt_number":1, # retry number + "outcome":"", # type(outcome.result()) = "str", type(outcome.exception()) = "class" + "outcome_timestamp":143.098416904, + "idle_for":0, + "next_action":"None" + } + """ + if retry_state.outcome.failed: + if retry_state.args: + # # can't be used as args=retry_state.args + func_param_output = retry_state.args[0] + elif retry_state.kwargs: + func_param_output = retry_state.kwargs.get("output", "") + exp_str = str(retry_state.outcome.exception()) + logger.warning( + f"parse json from content inside [CONTENT][/CONTENT] failed at retry " + f"{retry_state.attempt_number}, try to fix it, exp: {exp_str}" + ) + + repaired_output = repair_invalid_json(func_param_output, exp_str) + retry_state.kwargs["output"] = repaired_output + + return run_and_passon + + +@retry( + stop=stop_after_attempt(3 if CONFIG.repair_llm_output else 0), + wait=wait_fixed(1), + after=run_after_exp_and_passon_next_retry(logger), +) +def retry_parse_json_text(output: str) -> Union[list, dict]: + """ + repair the json-text situation like there are extra chars like [']', '}'] + + Warning + if CONFIG.repair_llm_output is False, retry _aask_v1 {x=3} times, and the retry_parse_json_text's retry not work + if CONFIG.repair_llm_output is True, the _aask_v1 and the retry_parse_json_text will loop for {x=3*3} times. + it's a two-layer retry cycle + """ + logger.debug(f"output to json decode:\n{output}") + + # if CONFIG.repair_llm_output is True, it will try to fix output until the retry break + parsed_data = CustomDecoder(strict=False).decode(output) + + return parsed_data + + +def extract_content_from_output(content: str, right_key: str = "[/CONTENT]"): + """extract xxx from [CONTENT](xxx)[/CONTENT] using regex pattern""" + + def re_extract_content(cont: str, pattern: str) -> str: + matches = re.findall(pattern, cont, re.DOTALL) + for match in matches: + if match: + cont = match + break + return cont.strip() + + # TODO construct the extract pattern with the `right_key` + raw_content = copy.deepcopy(content) + pattern = r"\[CONTENT\]([\s\S]*)\[/CONTENT\]" + new_content = re_extract_content(raw_content, pattern) + + if not new_content.startswith("{"): + # TODO find a more general pattern + # # for `[CONTENT]xxx[CONTENT]xxxx[/CONTENT] situation + logger.warning(f"extract_content try another pattern: {pattern}") + if right_key not in new_content: + raw_content = copy.deepcopy(new_content + "\n" + right_key) + # # pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" + new_content = re_extract_content(raw_content, pattern) + else: + if right_key in new_content: + idx = new_content.find(right_key) + new_content = new_content[:idx] + new_content = new_content.strip() + + return new_content + + +def extract_state_value_from_output(content: str) -> str: + """ + For openai models, they will always return state number. But for open llm models, the instruction result maybe a + long text contain target number, so here add a extraction to improve success rate. + + Args: + content (str): llm's output from `Role._think` + """ + content = content.strip() # deal the output cases like " 0", "0\n" and so on. + pattern = r"([0-9])" # TODO find the number using a more proper method not just extract from content using pattern + matches = re.findall(pattern, content, re.DOTALL) + matches = list(set(matches)) + state = matches[0] if len(matches) > 0 else "-1" + return state diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py new file mode 100644 index 000000000..5ceed65d9 --- /dev/null +++ b/metagpt/utils/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +import typing + +from tenacity import _utils + + +def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: + def log_it(retry_state: "RetryCallState") -> None: + if retry_state.fn is None: + fn_name = "" + else: + fn_name = _utils.get_callback_name(retry_state.fn) + logger.error( + f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " + f"exp: {retry_state.outcome.exception()}" + ) + + return log_it diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 49969a2af..408fd3162 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -33,5 +33,6 @@ async def test_llm_acompletion(llm): assert len(await llm.acompletion_batch([hello_msg])) > 0 assert len(await llm.acompletion_batch_text([hello_msg])) > 0 + # if __name__ == "__main__": # pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_ahttp_client.py b/tests/metagpt/utils/test_ahttp_client.py new file mode 100644 index 000000000..a595d645f --- /dev/null +++ b/tests/metagpt/utils/test_ahttp_client.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of ahttp_client + +import pytest + +from metagpt.utils.ahttp_client import apost, apost_stream + + +@pytest.mark.asyncio +async def test_apost(): + result = await apost(url="https://www.baidu.com/") + assert "百度一下" in result + + result = await apost( + url="http://aider.meizu.com/app/weather/listWeather", data={"cityIds": "101240101"}, as_json=True + ) + assert result["code"] == "200" + + +@pytest.mark.asyncio +async def test_apost_stream(): + result = apost_stream(url="https://www.baidu.com/") + async for line in result: + assert len(line) >= 0 + + result = apost_stream(url="http://aider.meizu.com/app/weather/listWeather", data={"cityIds": "101240101"}) + async for line in result: + assert len(line) >= 0 diff --git a/tests/metagpt/utils/test_custom_decoder.py b/tests/metagpt/utils/test_custom_decoder.py index c7b14ad59..4af7a6cdc 100644 --- a/tests/metagpt/utils/test_custom_decoder.py +++ b/tests/metagpt/utils/test_custom_decoder.py @@ -6,6 +6,7 @@ @File : test_custom_decoder.py """ +import pytest from metagpt.utils.custom_decoder import CustomDecoder @@ -37,6 +38,46 @@ def test_parse_single_quote(): parsed_data = decoder.decode(input_data) assert 'a"\n b' in parsed_data + input_data = """{ + 'a': " + b +" +} +""" + with pytest.raises(Exception): + parsed_data = decoder.decode(input_data) + + input_data = """{ + 'a': ' + b +' +} +""" + with pytest.raises(Exception): + parsed_data = decoder.decode(input_data) + + +def test_parse_double_quote(): + decoder = CustomDecoder(strict=False) + + input_data = """{ + "a": " + b +" +} +""" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "\n b\n" + + input_data = """{ + "a": ' + b +' +} +""" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "\n b\n" + def test_parse_triple_double_quote(): # Create a custom JSON decoder @@ -54,6 +95,10 @@ def test_parse_triple_double_quote(): parsed_data = decoder.decode(input_data) assert parsed_data["a"] == "b" + input_data = "{\"\"\"a\"\"\": '''b'''}" + parsed_data = decoder.decode(input_data) + assert parsed_data["a"] == "b" + def test_parse_triple_single_quote(): # Create a custom JSON decoder diff --git a/tests/metagpt/utils/test_repair_llm_raw_output.py b/tests/metagpt/utils/test_repair_llm_raw_output.py new file mode 100644 index 000000000..21bbee921 --- /dev/null +++ b/tests/metagpt/utils/test_repair_llm_raw_output.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of repair_llm_raw_output + + +from metagpt.config import CONFIG +from metagpt.utils.repair_llm_raw_output import ( + RepairType, + extract_content_from_output, + repair_invalid_json, + repair_llm_raw_output, + retry_parse_json_text, +) + +CONFIG.repair_llm_output = True + + +def test_repair_case_sensitivity(): + raw_output = """{ + "Original requirements": "Write a 2048 game", + "search Information": "", + "competitive Quadrant charT": "quadrantChart + Campaign A: [0.3, 0.6]", + "requirement analysis": "The 2048 game should be simple to play" +}""" + target_output = """{ + "Original Requirements": "Write a 2048 game", + "Search Information": "", + "Competitive Quadrant Chart": "quadrantChart + Campaign A: [0.3, 0.6]", + "Requirement Analysis": "The 2048 game should be simple to play" +}""" + req_keys = ["Original Requirements", "Search Information", "Competitive Quadrant Chart", "Requirement Analysis"] + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) + assert output == target_output + + +def test_repair_special_character_missing(): + raw_output = """[CONTENT] + "Anything UNCLEAR": "No unclear requirements or information." +[CONTENT]""" + + target_output = """[CONTENT] + "Anything UNCLEAR": "No unclear requirements or information." +[/CONTENT]""" + req_keys = ["[/CONTENT]"] + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) + assert output == target_output + + raw_output = """[CONTENT] tag +[CONTENT] +{ +"Anything UNCLEAR": "No unclear requirements or information." +} +[CONTENT]""" + target_output = """[CONTENT] tag +[CONTENT] +{ +"Anything UNCLEAR": "No unclear requirements or information." +} +[/CONTENT]""" + output = repair_llm_raw_output(output=raw_output, req_keys=req_keys) + assert output == target_output + + raw_output = '[CONTENT] {"a": "b"} [CONTENT]' + target_output = '[CONTENT] {"a": "b"} [/CONTENT]' + + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) + print("output\n", output) + assert output == target_output + + +def test_required_key_pair_missing(): + raw_output = '[CONTENT] {"a": "b"}' + target_output = '[CONTENT] {"a": "b"}\n[/CONTENT]' + + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) + assert output == target_output + + raw_output = """[CONTENT] +{ + "key": "value" +]""" + target_output = """[CONTENT] +{ + "key": "value" +] +[/CONTENT]""" + + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) + assert output == target_output + + raw_output = """[CONTENT] tag +[CONTENT] +{ + "key": "value" +} +xxx +""" + target_output = """[CONTENT] +{ + "key": "value" +} +[/CONTENT]""" + output = repair_llm_raw_output(output=raw_output, req_keys=["[/CONTENT]"]) + assert output == target_output + + +def test_repair_json_format(): + raw_output = "{ xxx }]" + target_output = "{ xxx }" + + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) + assert output == target_output + + raw_output = "[{ xxx }" + target_output = "{ xxx }" + + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) + assert output == target_output + + raw_output = "{ xxx ]" + target_output = "{ xxx }" + + output = repair_llm_raw_output(output=raw_output, req_keys=[None], repair_type=RepairType.JSON) + assert output == target_output + + +def test_repair_invalid_json(): + raw_output = """{ + "key": "value" + }, +}""" + target_output = """{ + "key": "value" +, +}""" + output = repair_invalid_json(raw_output, "Expecting ',' delimiter: line 3 column 1") + assert output == target_output + + raw_output = """{ + "key": " +value + }, +}""" + target_output = """{ + "key": " +value +", +}""" + output = repair_invalid_json(raw_output, "Expecting ',' delimiter: line 4 column 1") + output = repair_invalid_json(output, "Expecting ',' delimiter: line 4 column 1") + assert output == target_output + + raw_output = """{ + "key": ' +value + }, +}""" + target_output = """{ + "key": ' +value +', +}""" + output = repair_invalid_json(raw_output, "Expecting ',' delimiter: line 4 column 1") + output = repair_invalid_json(output, "Expecting ',' delimiter: line 4 column 1") + output = repair_invalid_json(output, "Expecting ',' delimiter: line 4 column 1") + assert output == target_output + + +def test_retry_parse_json_text(): + invalid_json_text = """{ +"Original Requirements": "Create a 2048 game", +"Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis" +], +"Requirement Analysis": "The requirements are clear and well-defined" +}""" + target_json = { + "Original Requirements": "Create a 2048 game", + "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", + "Requirement Analysis": "The requirements are clear and well-defined", + } + output = retry_parse_json_text(output=invalid_json_text) + assert output == target_json + + invalid_json_text = """{ +"Original Requirements": "Create a 2048 game", +"Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis" +}, +"Requirement Analysis": "The requirements are clear and well-defined" +}""" + target_json = { + "Original Requirements": "Create a 2048 game", + "Competitive Quadrant Chart": "quadrantChart\n\ttitle Reach and engagement of campaigns\n\t\tx-axis", + "Requirement Analysis": "The requirements are clear and well-defined", + } + output = retry_parse_json_text(output=invalid_json_text) + assert output == target_json + + +def test_extract_content_from_output(): + """ + cases + xxx [CONTENT] xxxx [/CONTENT] + xxx [CONTENT] xxx [CONTENT] xxxx [/CONTENT] + xxx [CONTENT] xxxx [/CONTENT] xxx [CONTENT][/CONTENT] xxx [CONTENT][/CONTENT] # target pair is the last one + """ + + output = ( + 'Sure! Here is the properly formatted JSON output based on the given context:\n\n[CONTENT]\n{\n"' + 'Required Python third-party packages": [\n"pygame==2.0.4",\n"pytest"\n],\n"Required Other language ' + 'third-party packages": [\n"No third-party packages are required."\n],\n"Full API spec": "\nopenapi: ' + "3.0.0\n\ndescription: A JSON object representing the game state.\n\npaths:\n game:\n get:\n " + "summary: Get the current game state.\n responses:\n 200:\n description: Game state." + "\n\n moves:\n post:\n summary: Make a move.\n requestBody:\n description: Move to be " + "made.\n content:\n applicationjson:\n schema:\n type: object\n " + " properties:\n x:\n type: integer\n y:\n " + " type: integer\n tile:\n type: object\n " + "properties:\n value:\n type: integer\n x:\n " + " type: integer\n y:\n type: integer\n\n " + "undo-move:\n post:\n summary: Undo the last move.\n responses:\n 200:\n " + " description: Undone move.\n\n end-game:\n post:\n summary: End the game.\n responses:\n " + " 200:\n description: Game ended.\n\n start-game:\n post:\n summary: Start a new " + "game.\n responses:\n 200:\n description: Game started.\n\n game-over:\n get:\n " + " summary: Check if the game is over.\n responses:\n 200:\n description: Game " + "over.\n 404:\n description: Game not over.\n\n score:\n get:\n summary: Get the " + "current score.\n responses:\n 200:\n description: Score.\n\n tile:\n get:\n " + "summary: Get a specific tile.\n parameters:\n tile_id:\n type: integer\n " + "description: ID of the tile to get.\n responses:\n 200:\n description: Tile.\n\n " + "tiles:\n get:\n summary: Get all tiles.\n responses:\n 200:\n description: " + "Tiles.\n\n level:\n get:\n summary: Get the current level.\n responses:\n 200:\n " + " description: Level.\n\n level-up:\n post:\n summary: Level up.\n responses:\n " + "200:\n description: Level up successful.\n\n level-down:\n post:\n summary: Level " + "down.\n responses:\n 200:\n description: Level down successful.\n\n restart:\n " + "post:\n summary: Restart the game.\n responses:\n 200:\n description: Game " + "restarted.\n\n help:\n get:\n summary: Get help.\n responses:\n 200:\n " + "description: Help.\n\n version:\n get:\n summary: Get the version of the game.\n " + 'responses:\n 200:\n description: Version.\n\n}\n\n"Logic Analysis": [\n"game.py",' + '\n"Contains the game logic."\n],\n"Task list": [\n"game.py",\n"Contains the game logic and should be ' + 'done first."\n],\n"Shared Knowledge": "\n\'game.py\' contains the game logic.\n",\n"Anything ' + 'UNCLEAR": "How to start the game."\n]\n\n[/CONTENT] Great! Your JSON output is properly formatted ' + "and correctly includes all the required sections. Here's a breakdown of what each section " + "contains:\n\nRequired Python third-party packages:\n\n* pygame==2.0.4\n* pytest\n\nRequired Other " + "language third-party packages:\n\n* No third-party packages are required.\n\nFull API spec:\n\n* " + "openapi: 3.0.0\n* description: A JSON object representing the game state.\n* paths:\n + game: " + "Get the current game state.\n + moves: Make a move.\n + undo-move: Undo the last move.\n + " + "end-game: End the game.\n + start-game: Start a new game.\n + game-over: Check if the game is " + "over.\n + score: Get the current score.\n + tile: Get a specific tile.\n + tiles: Get all tiles.\n " + "+ level: Get the current level.\n + level-up: Level up.\n + level-down: Level down.\n + restart: " + "Restart the game.\n + help: Get help.\n + version: Get the version of the game.\n\nLogic " + "Analysis:\n\n* game.py contains the game logic.\n\nTask list:\n\n* game.py contains the game logic " + "and should be done first.\n\nShared Knowledge:\n\n* 'game.py' contains the game logic.\n\nAnything " + "UNCLEAR:\n\n* How to start the game.\n\nGreat job! This JSON output should provide a clear and " + "comprehensive overview of the project's requirements and dependencies." + ) + output = extract_content_from_output(output) + assert output.startswith('{\n"Required Python third-party packages') and output.endswith( + 'UNCLEAR": "How to start the game."\n]' + ) + + output = ( + "Sure, I would be happy to help! Here is the information you provided, formatted as a JSON object " + 'inside the [CONTENT] tag:\n\n[CONTENT]\n{\n"Original Requirements": "Create a 2048 game",\n"Search ' + 'Information": "Search results for 2048 game",\n"Requirements": [\n"Create a game with the same rules ' + 'as the original 2048 game",\n"Implement a user interface that is easy to use and understand",\n"Add a ' + 'scoreboard to track the player progress",\n"Allow the player to undo and redo moves",\n"Implement a ' + 'game over screen to display the final score"\n],\n"Product Goals": [\n"Create a fun and engaging game ' + 'experience for the player",\n"Design a user interface that is visually appealing and easy to use",\n"' + 'Optimize the game for performance and responsiveness"\n],\n"User Stories": [\n"As a player, I want to ' + 'be able to move tiles around the board to combine numbers",\n"As a player, I want to be able to undo ' + 'and redo moves to correct mistakes",\n"As a player, I want to see the final score and game over screen' + ' when I win"\n],\n"Competitive Analysis": [\n"Competitor A: 2048 game with a simple user interface and' + ' basic graphics",\n"Competitor B: 2048 game with a more complex user interface and better graphics",' + '\n"Competitor C: 2048 game with a unique twist on the rules and a more challenging gameplay experience"' + '\n],\n"Competitive Quadrant Chart": "quadrantChart\\n\ttitle Reach and engagement of campaigns\\n\t\t' + "x-axis Low Reach --> High Reach\\n\t\ty-axis Low Engagement --> High Engagement\\n\tquadrant-1 We " + "should expand\\n\tquadrant-2 Need to promote\\n\tquadrant-3 Re-evaluate\\n\tquadrant-4 May be " + "improved\\n\tCampaign A: [0.3, 0.6]\\n\tCampaign B: [0.45, 0.23]\\n\tCampaign C: [0.57, 0.69]\\n\t" + 'Campaign D: [0.78, 0.34]\\n\tCampaign E: [0.40, 0.34]\\n\tCampaign F: [0.35, 0.78]"\n],\n"Requirement ' + 'Analysis": "The requirements are clear and well-defined, but there may be some ambiguity around the ' + 'specific implementation details",\n"Requirement Pool": [\n["P0", "Implement a game with the same ' + 'rules as the original 2048 game"],\n["P1", "Add a scoreboard to track the player progress"],\n["P2", ' + '"Allow the player to undo and redo moves"]\n],\n"UI Design draft": "The UI should be simple and easy ' + "to use, with a clean and visually appealing design. The game board should be the main focus of the " + 'UI, with clear and concise buttons for the player to interact with.",\n"Anything UNCLEAR": ""\n}\n' + "[/CONTENT]\n\nI hope this helps! Let me know if you have any further questions or if there anything " + "else I can do to assist you." + ) + output = extract_content_from_output(output) + assert output.startswith('{\n"Original Requirements"') and output.endswith('"Anything UNCLEAR": ""\n}') + + output = """ Sure, I'd be happy to help! Here's the JSON output for the given context:\n\n[CONTENT]\n{ +"Implementation approach": "We will use the open-source framework PyGame to create a 2D game engine, which will + provide us with a robust and efficient way to handle game logic and rendering. PyGame is widely used in the game + development community and has a large number of resources and tutorials available online.",\n"Python package name": + "pygame_2048",\n"File list": ["main.py", "game.py", "constants.py", "ui.py"],\n"Data structures and interface + definitions": '\nclassDiagram\n class Game{\n +int score\n +list tiles\n +function + move_tile(tile, int dx, int dy)\n +function undo_move()\n +function get_highest_score()\n }\n + class Tile{\n +int value\n +int x\n +int y\n }\n ...\n Game "1" -- "1" Food: has\n', +\n"Program call flow": '\nsequenceDiagram\n participant M as Main\n participant G as Game\n ...\n G->>M: + end game\n',\n"Anything UNCLEAR": "The requirement is clear to me."\n}\n[/CONTENT] Here's the JSON output for the + given context, wrapped inside the [CONTENT][/CONTENT] format:\n\n[CONTENT]\n{\n"Implementation approach": "We will + use the open-source framework PyGame to create a 2D game engine, which will provide us with a robust and efficient + way to handle game logic and rendering. PyGame is widely used in the game development community and has a large + number of resources and tutorials available online.",\n"Python package name": "pygame_2048",\n"File list": + ["main.py", "game.py", "constants.py", "ui.py"],\n"Data structures and interface definitions": '\nclassDiagram\n + class Game{\n +int score\n +list tiles\n +function move_tile(tile, int dx, int dy)\n ++function undo_move()\n +function get_highest_score()\n }\n class Tile{\n +int value\n +int x\n + +int y\n }\n ...\n Game "1" -- "1" Food: has\n',\n"Program call flow": '\nsequenceDiagram\n participant + M as Main\n participant G as Game\n ...\n G->>M: end game\n',\n"Anything UNCLEAR": "The requirement is + clear to me."\n}\n[/CONTENT] Great! Your JSON output is well-formatted and provides all the necessary + information for a developer to understand the design and implementation of the 2048 game. +""" + output = extract_content_from_output(output) + assert output.startswith('{\n"Implementation approach"') and output.endswith( + '"Anything UNCLEAR": "The requirement is clear to me."\n}' + )