Merge pull request #561 from iorisa/merge/geekan/main_to_env_refactor

Merge/geekan/main to env refactor

.gitignore

@@ -59,6 +59,7 @@ cover/
 
 # Django stuff:
 *.log
+logs
 local_settings.py
 db.sqlite3
 db.sqlite3-journal

config/config.yaml

@@ -34,6 +34,15 @@ RPM: 10
 #### if zhipuai from `https://open.bigmodel.cn`. You can set here or export API_KEY="YOUR_API_KEY"
 # ZHIPUAI_API_KEY: "YOUR_API_KEY"
 
+#### if use self-host open llm model with openai-compatible interface
+#OPEN_LLM_API_BASE: "http://127.0.0.1:8000/v1"
+#OPEN_LLM_API_MODEL: "llama2-13b"
+#
+##### if use Fireworks api
+#FIREWORKS_API_KEY: "YOUR_API_KEY"
+#FIREWORKS_API_BASE: "https://api.fireworks.ai/inference/v1"
+#FIREWORKS_API_MODEL: "YOUR_LLM_MODEL"  # example, accounts/fireworks/models/llama-v2-13b-chat
+
 #### for Search
 
 ## Supported values: serpapi/google/serper/ddg
@@ -94,4 +103,9 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k
 ### browser path for pyppeteer engine, support Chrome, Chromium,MS Edge
 #PYPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable"
 
+### for repair non-openai LLM's output when parse json-text if PROMPT_FORMAT=json
+### due to non-openai LLM's output will not always follow the instruction, so here activate a post-process
+### repair operation on the content extracted from LLM's raw output. Warning, it improves the result but not fix all cases.
+# REPAIR_LLM_OUTPUT: false
+
 PROMPT_FORMAT: json #json or markdown

metagpt/actions/action.py

@@ -5,7 +5,7 @@
 @Author  : alexanderwu
 @File    : action.py
 """
-import re
+
 from abc import ABC
 from typing import Optional
 
@@ -14,8 +14,9 @@
 from metagpt.actions.action_output import ActionOutput
 from metagpt.llm import LLM
 from metagpt.logs import logger
+from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess
 from metagpt.utils.common import OutputParser
-from metagpt.utils.custom_decoder import CustomDecoder
+from metagpt.utils.utils import general_after_log
 
 
 class Action(ABC):
@@ -57,7 +58,11 @@ async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> s
         system_msgs.append(self.prefix)
         return await self.llm.aask(prompt, system_msgs)
 
-    @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
+    @retry(
+        wait=wait_random_exponential(min=1, max=60),
+        stop=stop_after_attempt(6),
+        after=general_after_log(logger),
+    )
     async def _aask_v1(
         self,
         prompt: str,
@@ -67,24 +72,16 @@ async def _aask_v1(
         format="markdown",  # compatible to original format
     ) -> ActionOutput:
         content = await self.llm.aask(prompt, system_msgs)
-        logger.debug(content)
+        logger.debug(f"llm raw output:\n{content}")
         output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping)
 
         if format == "json":
-            pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]"
-            matches = re.findall(pattern, content, re.DOTALL)
-
-            for match in matches:
-                if match:
-                    content = match
-                    break
-
-            parsed_data = CustomDecoder(strict=False).decode(content)
+            parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]")
 
         else:  # using markdown parser
             parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping)
 
-        logger.debug(parsed_data)
+        logger.debug(f"parsed_data:\n{parsed_data}")
         instruct_content = output_class(**parsed_data)
         return ActionOutput(content, instruct_content)
 

metagpt/actions/action_node.py

@@ -5,13 +5,12 @@
 @Author  : alexanderwu
 @File    : action_node.py
 """
-import re
-from typing import Dict, Type, List, Any, Tuple, Optional
 import json
+import re
+from typing import Any, Dict, List, Optional, Type
 
 from pydantic import BaseModel, create_model, root_validator, validator
-#    , model_validator, field_validator
-from tenacity import wait_random_exponential, stop_after_attempt, retry
+from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions import ActionOutput
 from metagpt.llm import BaseGPTAPI
@@ -53,6 +52,7 @@ def dict_to_markdown(d, prefix="-", postfix="\n"):
 
 class ActionNode:
     """ActionNode is a tree of nodes."""
+
     # Action Strgy
     # - sop: 仅使用一级SOP
     # - complex: 使用一级SOP+自定义策略填槽
@@ -74,8 +74,7 @@ class ActionNode:
     content: str
     instruct_content: BaseModel
 
-    def __init__(self, key, expected_type, instruction, example, content="",
-                 children=None):
+    def __init__(self, key, expected_type, instruction, example, content="", children=None):
         self.key = key
         self.expected_type = expected_type
         self.instruction = instruction
@@ -84,8 +83,9 @@ def __init__(self, key, expected_type, instruction, example, content="",
         self.children = children if children is not None else {}
 
     def __str__(self):
-        return f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" \
-               f", {self.content}, {self.children}"
+        return (
+            f"{self.key}, {self.expected_type}, {self.instruction}, {self.example}" f", {self.content}, {self.children}"
+        )
 
     def __repr__(self):
         return self.__str__()
@@ -116,7 +116,7 @@ def get_self_mapping(self) -> Dict[str, Type]:
 
     def get_mapping(self, mode="children") -> Dict[str, Type]:
         """get key: type mapping under mode"""
-        if mode == "children" or (mode=="auto" and self.children):
+        if mode == "children" or (mode == "auto" and self.children):
             return self.get_children_mapping()
         return self.get_self_mapping()
 
@@ -148,15 +148,15 @@ def create_model_class_v2(cls, class_name: str, mapping: Dict[str, Type]):
         """基于pydantic v2的模型动态生成，用来检验结果类型正确性，待验证"""
         new_class = create_model(class_name, **mapping)
 
-        @model_validator(mode='before')
+        @model_validator(mode="before")
         def check_missing_fields(data):
             required_fields = set(mapping.keys())
             missing_fields = required_fields - set(data.keys())
             if missing_fields:
                 raise ValueError(f"Missing fields: {missing_fields}")
             return data
 
-        @field_validator('*')
+        @field_validator("*")
         def check_name(v: Any, field: str) -> Any:
             if field not in mapping.keys():
                 raise ValueError(f"Unrecognized block: {field}")
@@ -242,8 +242,9 @@ def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE)
         # FIXME: json instruction会带来格式问题，如："Project name": "web_2048  # 项目名称使用下划线",
         self.instruction = self.compile_instruction(to="markdown", mode=mode)
         self.example = self.compile_example(to=to, tag="CONTENT", mode=mode)
-        prompt = template.format(context=context, example=self.example, instruction=self.instruction,
-                                 constraint=CONSTRAINT)
+        prompt = template.format(
+            context=context, example=self.example, instruction=self.instruction, constraint=CONSTRAINT
+        )
         return prompt
 
     @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
@@ -302,7 +303,7 @@ async def simple_fill(self, to, mode):
         return self
 
     async def fill(self, context, llm, to="json", mode="auto", strgy="simple"):
-        """ Fill the node(s) with mode.
+        """Fill the node(s) with mode.
 
         :param context: Everything we should know when filling node.
         :param llm: Large Language Model with pre-defined system message.
@@ -336,15 +337,13 @@ async def fill(self, context, llm, to="json", mode="auto", strgy="simple"):
 
 def action_node_from_tuple_example():
     # 示例：列表中包含元组
-    list_of_tuples = [
-        ("key1", str, "Instruction 1", "Example 1")
-    ]
+    list_of_tuples = [("key1", str, "Instruction 1", "Example 1")]
 
     # 从列表中创建 ActionNode 实例
     nodes = [ActionNode(*data) for data in list_of_tuples]
     for i in nodes:
         logger.info(i)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     action_node_from_tuple_example()

metagpt/actions/debug_error.py

@@ -42,7 +42,7 @@
 ```
 ---
 Now you should start rewriting the code:
-## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE.
+## file name of the code to rewrite: Write code with triple quote. Do your best to implement THIS IN ONLY ONE FILE.
 """
 
 

metagpt/actions/design_api.py

@@ -11,7 +11,6 @@
 """
 import json
 from pathlib import Path
-# from typing import List
 
 from metagpt.actions import Action, ActionOutput
 from metagpt.actions.design_api_an import DESIGN_API_NODE
@@ -26,9 +25,13 @@
 from metagpt.logs import logger
 from metagpt.schema import Document, Documents
 from metagpt.utils.file_repository import FileRepository
+
 # from metagpt.utils.get_template import get_template
 from metagpt.utils.mermaid import mermaid_to_file
 
+# from typing import List
+
+
 NEW_REQ_TEMPLATE = """
 ### Legacy Content
 {old_design}
@@ -82,9 +85,7 @@ async def _new_system_design(self, context, format=CONFIG.prompt_format):
         return node
 
     async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format):
-        context = NEW_REQ_TEMPLATE.format(
-            old_design=system_design_doc.content, context=prd_doc.content
-        )
+        context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content)
         node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format)
         system_design_doc.content = node.instruct_content.json(ensure_ascii=False)
         return system_design_doc

metagpt/actions/design_api_an.py

@@ -6,52 +6,49 @@
 @File    : design_api_an.py
 """
 from metagpt.actions.action_node import ActionNode
-from metagpt.utils.mermaid import MMC1, MMC2
 from metagpt.logs import logger
+from metagpt.utils.mermaid import MMC1, MMC2
 
 IMPLEMENTATION_APPROACH = ActionNode(
     key="Implementation approach",
     expected_type=str,
     instruction="Analyze the difficult points of the requirements, select the appropriate open-source framework",
-    example="We will ..."
+    example="We will ...",
 )
 
 PROJECT_NAME = ActionNode(
-    key="Project name",
-    expected_type=str,
-    instruction="The project name with underline",
-    example="game_2048"
+    key="Project name", expected_type=str, instruction="The project name with underline", example="game_2048"
 )
 
 FILE_LIST = ActionNode(
     key="File list",
     expected_type=list[str],
     instruction="Only need relative paths. ALWAYS write a main.py or app.py here",
-    example=['main.py', 'game.py']
+    example=["main.py", "game.py"],
 )
 
 DATA_STRUCTURES_AND_INTERFACES = ActionNode(
     key="Data structures and interfaces",
     expected_type=str,
     instruction="Use mermaid classDiagram code syntax, including classes, method(__init__ etc.) and functions with type"
-                " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. "
-                "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.",
-    example=MMC1
+    " annotations, CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. "
+    "The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.",
+    example=MMC1,
 )
 
 PROGRAM_CALL_FLOW = ActionNode(
     key="Program call flow",
     expected_type=str,
     instruction="Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE "
-                "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.",
-    example=MMC2
+    "accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.",
+    example=MMC2,
 )
 
 ANYTHING_UNCLEAR = ActionNode(
     key="Anything UNCLEAR",
     expected_type=str,
     instruction="Mention unclear project aspects, then try to clarify it.",
-    example="Clarification needed on third-party API integration, ..."
+    example="Clarification needed on third-party API integration, ...",
 )
 
 NODES = [
@@ -60,7 +57,7 @@
     FILE_LIST,
     DATA_STRUCTURES_AND_INTERFACES,
     PROGRAM_CALL_FLOW,
-    ANYTHING_UNCLEAR
+    ANYTHING_UNCLEAR,
 ]
 
 DESIGN_API_NODE = ActionNode.from_children("DesignAPI", NODES)
@@ -71,5 +68,5 @@ def main():
     logger.info(prompt)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

metagpt/actions/project_management.py

@@ -10,7 +10,6 @@
         3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality.
 """
 import json
-# from typing import List
 
 from metagpt.actions import ActionOutput
 from metagpt.actions.action import Action
@@ -25,6 +24,9 @@
 from metagpt.logs import logger
 from metagpt.schema import Document, Documents
 from metagpt.utils.file_repository import FileRepository
+
+# from typing import List
+
 # from metagpt.utils.get_template import get_template
 
 NEW_REQ_TEMPLATE = """
@@ -97,7 +99,8 @@ async def _run_new_tasks(self, context, format=CONFIG.prompt_format):
     async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document:
         context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content)
         node = await PM_NODE.fill(context, self.llm, format)
-        return node
+        task_doc.content = node.content
+        return task_doc
 
     @staticmethod
     async def _update_requirements(doc):