Merge pull request #688 from iorisa/feature/rebuild_class_view

fixbug: invalid default project name

README.md

@@ -34,7 +34,10 @@
 <p align="center">Software Company Multi-Role Schematic (Gradually Implementing)</p>
 
 ## News
-- Dec 15: [v0.5.0](https://github.com/geekan/MetaGPT/releases/tag/v0.5.0) is released! We introduce **incremental development**, facilitating agents to build up larger projects on top of their previous efforts or existing codebase. We also launch a whole collection of important features, including **multilingual support** (experimental), multiple **programming languages support** (experimental), **incremental development** (experimental), CLI support, pip support, enhanced code review, documentation mechanism, and optimized messaging mechanism!
+🚀 Jan 03: Here comes [v0.6.0](https://github.com/geekan/MetaGPT/releases/tag/v0.6.0)! In this version, we added serialization and deserialization of important objects and enabled breakpoint recovery. We upgraded OpenAI package to v1.6.0 and supported Gemini, ZhipuAI, Ollama, OpenLLM, etc. Moreover, we provided extremely simple examples where you need only 7 lines to implement a general election [debate](https://github.com/geekan/MetaGPT/blob/main/examples/debate_simple.py). Check out more details [here](https://github.com/geekan/MetaGPT/releases/tag/v0.6.0)!
+
+
+🚀 Dec 15: [v0.5.0](https://github.com/geekan/MetaGPT/releases/tag/v0.5.0) is released! We introduced **incremental development**, facilitating agents to build up larger projects on top of their previous efforts or existing codebase. We also launched a whole collection of important features, including **multilingual support** (experimental), multiple **programming languages support** (experimental), **incremental development** (experimental), CLI support, pip support, enhanced code review, documentation mechanism, and optimized messaging mechanism!
 
 ## Install
 

config/config.yaml

@@ -15,6 +15,7 @@ OPENAI_API_MODEL: "gpt-4-1106-preview"
 MAX_TOKENS: 4096
 RPM: 10
 TIMEOUT: 60 # Timeout for llm invocation
+#DEFAULT_PROVIDER: openai
 
 #### if Spark
 #SPARK_APPID : "YOUR_APPID"

docs/.agent-store-config.yaml.example

@@ -1,7 +1,7 @@
 role:
   name: Teacher  # Referenced the `Teacher` in `metagpt/roles/teacher.py`.
   module: metagpt.roles.teacher  # Referenced `metagpt/roles/teacher.py`.
-  skills:  # Refer to the skill `name` of the published skill in `.well-known/skills.yaml`.
+  skills:  # Refer to the skill `name` of the published skill in `docs/.well-known/skills.yaml`.
     - name: text_to_speech
       description: Text-to-speech
     - name: text_to_image

metagpt/actions/prepare_documents.py

@@ -35,7 +35,6 @@ def _init_repo(self):
         if path.exists() and not CONFIG.inc:
             shutil.rmtree(path)
         CONFIG.project_path = path
-        CONFIG.project_name = path.name
         CONFIG.git_repo = GitRepository(local_path=path, auto_init=True)
 
     async def run(self, with_messages, **kwargs):

metagpt/actions/rebuild_class_view.py

@@ -33,11 +33,16 @@ async def run(self, with_messages=None, format=CONFIG.prompt_schema):
         graph_repo_pathname = CONFIG.git_repo.workdir / GRAPH_REPO_FILE_REPO / CONFIG.git_repo.workdir.name
         graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
         repo_parser = RepoParser(base_directory=Path(self.context))
-        class_views, relationship_views = await repo_parser.rebuild_class_views(path=Path(self.context))  # use pylint
+        # use pylint
+        class_views, relationship_views, package_root = await repo_parser.rebuild_class_views(path=Path(self.context))
         await GraphRepository.update_graph_db_with_class_views(graph_db, class_views)
         await GraphRepository.update_graph_db_with_class_relationship_views(graph_db, relationship_views)
-        symbols = repo_parser.generate_symbols()  # use ast
+        # use ast
+        direction, diff_path = self._diff_path(path_root=Path(self.context).resolve(), package_root=package_root)
+        symbols = repo_parser.generate_symbols()
         for file_info in symbols:
+            # Align to the same root directory in accordance with `class_views`.
+            file_info.file = self._align_root(file_info.file, direction, diff_path)
             await GraphRepository.update_graph_db_with_file_info(graph_db, file_info)
         await self._create_mermaid_class_views(graph_db=graph_db)
         await graph_db.save()
@@ -193,3 +198,20 @@ async def _parse_function_args(method: ClassMethod, ns_name: str, graph_db: Grap
                 method.args.append(ClassAttribute(name=parts[0].strip()))
                 continue
             method.args.append(ClassAttribute(name=parts[0].strip(), value_type=parts[-1].strip()))
+
+    @staticmethod
+    def _diff_path(path_root: Path, package_root: Path) -> (str, str):
+        if len(str(path_root)) > len(str(package_root)):
+            return "+", str(path_root.relative_to(package_root))
+        if len(str(path_root)) < len(str(package_root)):
+            return "-", str(package_root.relative_to(path_root))
+        return "=", "."
+
+    @staticmethod
+    def _align_root(path: str, direction: str, diff_path: str):
+        if direction == "=":
+            return path
+        if direction == "+":
+            return diff_path + "/" + path
+        else:
+            return path[len(diff_path) + 1 :]

metagpt/actions/rebuild_sequence_view.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/1/4
+@Author  : mashenquan
+@File    : rebuild_sequence_view.py
+@Desc    : Rebuild sequence view info
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List
+
+from metagpt.actions import Action
+from metagpt.config import CONFIG
+from metagpt.const import GRAPH_REPO_FILE_REPO
+from metagpt.logs import logger
+from metagpt.utils.common import aread, list_files
+from metagpt.utils.di_graph_repository import DiGraphRepository
+from metagpt.utils.graph_repository import GraphKeyword
+
+
+class RebuildSequenceView(Action):
+    async def run(self, with_messages=None, format=CONFIG.prompt_schema):
+        graph_repo_pathname = CONFIG.git_repo.workdir / GRAPH_REPO_FILE_REPO / CONFIG.git_repo.workdir.name
+        graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
+        entries = await RebuildSequenceView._search_main_entry(graph_db)
+        for entry in entries:
+            await self._rebuild_sequence_view(entry, graph_db)
+        await graph_db.save()
+
+    @staticmethod
+    async def _search_main_entry(graph_db) -> List:
+        rows = await graph_db.select(predicate=GraphKeyword.HAS_PAGE_INFO)
+        tag = "__name__:__main__"
+        entries = []
+        for r in rows:
+            if tag in r.subject or tag in r.object_:
+                entries.append(r)
+        return entries
+
+    async def _rebuild_sequence_view(self, entry, graph_db):
+        filename = entry.subject.split(":", 1)[0]
+        src_filename = RebuildSequenceView._get_full_filename(root=self.context, pathname=filename)
+        content = await aread(filename=src_filename, encoding="utf-8")
+        content = f"```python\n{content}\n```\n\n---\nTranslate the code above into Mermaid Sequence Diagram."
+        data = await self.llm.aask(
+            msg=content, system_msgs=["You are a python code to Mermaid Sequence Diagram translator in function detail"]
+        )
+        await graph_db.insert(subject=filename, predicate=GraphKeyword.HAS_SEQUENCE_VIEW, object_=data)
+        logger.info(data)
+
+    @staticmethod
+    def _get_full_filename(root: str | Path, pathname: str | Path) -> Path | None:
+        files = list_files(root=root)
+        postfix = "/" + str(pathname)
+        for i in files:
+            if str(i).endswith(postfix):
+                return i
+        return None

metagpt/actions/rebuild_sequence_view_an.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/1/4
+@Author  : mashenquan
+@File    : rebuild_sequence_view_an.py
+"""
+from metagpt.actions.action_node import ActionNode
+from metagpt.utils.mermaid import MMC2
+
+CODE_2_MERMAID_SEQUENCE_DIAGRAM = ActionNode(
+    key="Program call flow",
+    expected_type=str,
+    instruction='Translate the "context" content into "format example" format.',
+    example=MMC2,
+)

metagpt/actions/write_prd.py

@@ -14,6 +14,7 @@
 from __future__ import annotations
 
 import json
+import uuid
 from pathlib import Path
 from typing import Optional
 
@@ -117,7 +118,7 @@ async def _run_new_requirement(self, requirements, schema=CONFIG.prompt_schema)
         # if sas.result:
         #     logger.info(sas.result)
         #     logger.info(rsp)
-        project_name = CONFIG.project_name if CONFIG.project_name else ""
+        project_name = CONFIG.project_name or ""
         context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name)
         exclude = [PROJECT_NAME.key] if project_name else []
         node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, exclude=exclude)  # schema=schema
@@ -183,6 +184,8 @@ async def _rename_workspace(prd):
                 ws_name = CodeParser.parse_str(block="Project Name", text=prd)
             if ws_name:
                 CONFIG.project_name = ws_name
+        if not CONFIG.project_name:  # The LLM failed to provide a project name, and the user didn't provide one either.
+            CONFIG.project_name = "app" + uuid.uuid4().hex[:16]
         CONFIG.git_repo.rename_root(CONFIG.project_name)
 
     async def _is_bugfix(self, context) -> bool:

metagpt/actions/write_test.py

@@ -11,7 +11,6 @@
 from typing import Optional
 
 from metagpt.actions.action import Action
-from metagpt.config import CONFIG
 from metagpt.const import TEST_CODES_FILE_REPO
 from metagpt.logs import logger
 from metagpt.schema import Document, TestingContext
@@ -60,11 +59,12 @@ async def run(self, *args, **kwargs) -> TestingContext:
             self.context.test_doc = Document(
                 filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO
             )
+        fake_root = "/data"
         prompt = PROMPT_TEMPLATE.format(
             code_to_test=self.context.code_doc.content,
             test_file_name=self.context.test_doc.filename,
-            source_file_path=self.context.code_doc.root_relative_path,
-            workspace=CONFIG.git_repo.workdir,
+            source_file_path=fake_root + "/" + self.context.code_doc.root_relative_path,
+            workspace=fake_root,
         )
         self.context.test_doc.content = await self.write_code(prompt)
         return self.context

metagpt/config.py

@@ -50,6 +50,9 @@ class LLMProviderEnum(Enum):
     AZURE_OPENAI = "azure_openai"
     OLLAMA = "ollama"
 
+    def __missing__(self, key):
+        return self.OPENAI
+
 
 class Config(metaclass=Singleton):
     """
@@ -108,6 +111,11 @@ def get_default_llm_provider_enum(self) -> LLMProviderEnum:
             if v:
                 provider = k
                 break
+        if provider is None:
+            if self.DEFAULT_PROVIDER:
+                provider = LLMProviderEnum(self.DEFAULT_PROVIDER)
+            else:
+                raise NotConfiguredException("You should config a LLM configuration first")
 
         if provider is LLMProviderEnum.GEMINI and not require_python_version(req_version=(3, 10)):
             warnings.warn("Use Gemini requires Python >= 3.10")
@@ -117,7 +125,6 @@ def get_default_llm_provider_enum(self) -> LLMProviderEnum:
         if provider:
             logger.info(f"API: {provider}")
             return provider
-        raise NotConfiguredException("You should config a LLM configuration first")
 
     def get_model_name(self, provider=None) -> str:
         provider = provider or self.get_default_llm_provider_enum()

metagpt/learn/skill_loader.py

@@ -67,7 +67,7 @@ class SkillsDeclaration(BaseModel):
     @staticmethod
     async def load(skill_yaml_file_name: Path = None) -> "SkillsDeclaration":
         if not skill_yaml_file_name:
-            skill_yaml_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml"
+            skill_yaml_file_name = Path(__file__).parent.parent.parent / "docs/.well-known/skills.yaml"
         async with aiofiles.open(str(skill_yaml_file_name), mode="r") as reader:
             data = await reader.read(-1)
         skill_data = yaml.safe_load(data)

metagpt/repo_parser.py

@@ -240,12 +240,12 @@ async def rebuild_class_views(self, path: str | Path = None):
         class_views = await self._parse_classes(class_view_pathname)
         relationship_views = await self._parse_class_relationships(class_view_pathname)
         packages_pathname = path / "packages.dot"
-        class_views, relationship_views = RepoParser._repair_namespaces(
+        class_views, relationship_views, package_root = RepoParser._repair_namespaces(
             class_views=class_views, relationship_views=relationship_views, path=path
         )
         class_view_pathname.unlink(missing_ok=True)
         packages_pathname.unlink(missing_ok=True)
-        return class_views, relationship_views
+        return class_views, relationship_views, package_root
 
     async def _parse_classes(self, class_view_pathname):
         class_views = []
@@ -364,9 +364,9 @@ def _create_path_mapping(path: str | Path) -> Dict[str, str]:
     @staticmethod
     def _repair_namespaces(
         class_views: List[ClassInfo], relationship_views: List[ClassRelationship], path: str | Path
-    ) -> (List[ClassInfo], List[ClassRelationship]):
+    ) -> (List[ClassInfo], List[ClassRelationship], str):
         if not class_views:
-            return []
+            return [], [], ""
         c = class_views[0]
         full_key = str(path).lstrip("/").replace("/", ".")
         root_namespace = RepoParser._find_root(full_key, c.package)
@@ -388,7 +388,7 @@ def _repair_namespaces(
             v.src = RepoParser._repair_ns(v.src, new_mappings)
             v.dest = RepoParser._repair_ns(v.dest, new_mappings)
             relationship_views[i] = v
-        return class_views, relationship_views
+        return class_views, relationship_views, root_path
 
     @staticmethod
     def _repair_ns(package, mappings):

metagpt/utils/common.py

@@ -550,3 +550,20 @@ async def read_file_block(filename: str | Path, lineno: int, end_lineno: int):
                 break
             lines.append(line)
     return "".join(lines)
+
+
+def list_files(root: str | Path) -> List[Path]:
+    files = []
+    try:
+        directory_path = Path(root)
+        if not directory_path.exists():
+            return []
+        for file_path in directory_path.iterdir():
+            if file_path.is_file():
+                files.append(file_path)
+            else:
+                subfolder_files = list_files(root=file_path)
+                files.extend(subfolder_files)
+    except Exception as e:
+        logger.error(f"Error: {e}")
+    return files

metagpt/utils/graph_repository.py

@@ -135,12 +135,12 @@ async def update_graph_db_with_file_info(graph_db: "GraphRepository", file_info:
     @staticmethod
     async def update_graph_db_with_class_views(graph_db: "GraphRepository", class_views: List[ClassInfo]):
         for c in class_views:
-            filename, class_name = c.package.split(":", 1)
+            filename, _ = c.package.split(":", 1)
             await graph_db.insert(subject=filename, predicate=GraphKeyword.IS, object_=GraphKeyword.SOURCE_CODE)
             file_types = {".py": "python", ".js": "javascript"}
             file_type = file_types.get(Path(filename).suffix, GraphKeyword.NULL)
             await graph_db.insert(subject=filename, predicate=GraphKeyword.IS, object_=file_type)
-            await graph_db.insert(subject=filename, predicate=GraphKeyword.HAS_CLASS, object_=class_name)
+            await graph_db.insert(subject=filename, predicate=GraphKeyword.HAS_CLASS, object_=c.package)
             await graph_db.insert(
                 subject=c.package,
                 predicate=GraphKeyword.IS,

requirements.txt

@@ -22,7 +22,7 @@ pandas==2.0.3
 pydantic==2.5.3
 #pygame==2.1.3
 #pymilvus==2.2.8
-pytest==7.2.2
+# pytest==7.2.2 # test extras require
 python_docx==0.8.11
 PyYAML==6.0.1
 # sentence_transformers==2.2.2
@@ -38,7 +38,7 @@ typing-inspect==0.8.0
 typing_extensions==4.9.0
 libcst==1.0.1
 qdrant-client==1.7.0
-pytest-mock==3.11.1
+# pytest-mock==3.11.1  # test extras require
 # open-interpreter==0.1.7; python_version>"3.9" # Conflict with openai 1.x
 ta==0.10.2
 semantic-kernel==0.4.3.dev0
@@ -57,5 +57,5 @@ gitignore-parser==0.1.9
 websockets~=12.0
 networkx~=3.2.1
 google-generativeai==0.3.2
-playwright==1.40.0
+# playwright==1.40.0  # playwright extras require
 anytree

setup.py

@@ -46,6 +46,7 @@ def run(self):
     "chromadb==0.4.14",
     "gradio==3.0.0",
     "grpcio-status==1.48.2",
+    "mock==5.1.0",
 ]
 
 extras_require["pyppeteer"] = [
@@ -56,7 +57,7 @@ def run(self):
 
 setup(
     name="metagpt",
-    version="0.5.2",
+    version="0.6.0",
     description="The Multi-Agent Framework",
     long_description=long_description,
     long_description_content_type="text/markdown",