Merge pull request #676 from iorisa/feature/rebuild_class_view

feat: +rebuild class view

.gitignore

@@ -172,6 +172,8 @@ tests/metagpt/utils/file_repo_git
 *.png
 htmlcov
 htmlcov.*
+*.dot
 *.pkl
 *-structure.csv
 *-structure.json
+

docs/.well-known/openapi.yaml

@@ -11,7 +11,7 @@ paths:
     post:
       summary: Generate greeting
       description: Generates a greeting message.
-      operationId: hello.post_greeting
+      operationId: openapi_v3_hello.post_greeting
       responses:
         200:
           description: greeting response

metagpt/actions/rebuild_class_view.py

@@ -9,60 +9,187 @@
 import re
 from pathlib import Path
 
+import aiofiles
+
 from metagpt.actions import Action
 from metagpt.config import CONFIG
-from metagpt.const import CLASS_VIEW_FILE_REPO, GRAPH_REPO_FILE_REPO
+from metagpt.const import (
+    AGGREGATION,
+    COMPOSITION,
+    DATA_API_DESIGN_FILE_REPO,
+    GENERALIZATION,
+    GRAPH_REPO_FILE_REPO,
+)
+from metagpt.logs import logger
 from metagpt.repo_parser import RepoParser
+from metagpt.schema import ClassAttribute, ClassMethod, ClassView
+from metagpt.utils.common import split_namespace
 from metagpt.utils.di_graph_repository import DiGraphRepository
 from metagpt.utils.graph_repository import GraphKeyword, GraphRepository
 
 
 class RebuildClassView(Action):
-    def __init__(self, name="", context=None, llm=None):
-        super().__init__(name=name, context=context, llm=llm)
-
     async def run(self, with_messages=None, format=CONFIG.prompt_schema):
         graph_repo_pathname = CONFIG.git_repo.workdir / GRAPH_REPO_FILE_REPO / CONFIG.git_repo.workdir.name
         graph_db = await DiGraphRepository.load_from(str(graph_repo_pathname.with_suffix(".json")))
-        repo_parser = RepoParser(base_directory=self.context)
-        class_views = await repo_parser.rebuild_class_views(path=Path(self.context))  # use pylint
+        repo_parser = RepoParser(base_directory=Path(self.context))
+        class_views, relationship_views = await repo_parser.rebuild_class_views(path=Path(self.context))  # use pylint
         await GraphRepository.update_graph_db_with_class_views(graph_db, class_views)
+        await GraphRepository.update_graph_db_with_class_relationship_views(graph_db, relationship_views)
         symbols = repo_parser.generate_symbols()  # use ast
         for file_info in symbols:
             await GraphRepository.update_graph_db_with_file_info(graph_db, file_info)
-        await self._create_mermaid_class_view(graph_db=graph_db)
-        await self._save(graph_db=graph_db)
-
-    async def _create_mermaid_class_view(self, graph_db):
-        pass
-        # dataset = await graph_db.select(subject=concat_namespace(filename, class_name), predicate=GraphKeyword.HAS_PAGE_INFO)
-        # if not dataset:
-        #     logger.warning(f"No page info for {concat_namespace(filename, class_name)}")
-        #     return
-        # code_block_info = CodeBlockInfo.parse_raw(dataset[0].object_)
-        # src_code = await read_file_block(filename=Path(self.context) / filename, lineno=code_block_info.lineno, end_lineno=code_block_info.end_lineno)
-        # code_type = ""
-        # dataset = await graph_db.select(subject=filename, predicate=GraphKeyword.IS)
-        # for spo in dataset:
-        #     if spo.object_ in ["javascript", "python"]:
-        #         code_type = spo.object_
-        #         break
-
-        # try:
-        #     node = await REBUILD_CLASS_VIEW_NODE.fill(context=f"```{code_type}\n{src_code}\n```", llm=self.llm, to=format)
-        #     class_view = node.instruct_content.model_dump()["Class View"]
-        # except Exception as e:
-        #     class_view = RepoParser.rebuild_class_view(src_code, code_type)
-        # await graph_db.insert(subject=concat_namespace(filename, class_name), predicate=GraphKeyword.HAS_CLASS_VIEW, object_=class_view)
-        # logger.info(f"{concat_namespace(filename, class_name)} {GraphKeyword.HAS_CLASS_VIEW} {class_view}")
-
-    async def _save(self, graph_db):
-        class_view_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CLASS_VIEW_FILE_REPO)
-        dataset = await graph_db.select(predicate=GraphKeyword.HAS_CLASS_VIEW)
-        all_class_view = []
-        for spo in dataset:
-            title = f"---\ntitle: {spo.subject}\n---\n"
-            filename = re.sub(r"[/:]", "_", spo.subject) + ".mmd"
-            await class_view_file_repo.save(filename=filename, content=title + spo.object_)
-            all_class_view.append(spo.object_)
-        await class_view_file_repo.save(filename="all.mmd", content="\n".join(all_class_view))
+        await self._create_mermaid_class_views(graph_db=graph_db)
+        await graph_db.save()
+
+    async def _create_mermaid_class_views(self, graph_db):
+        path = Path(CONFIG.git_repo.workdir) / DATA_API_DESIGN_FILE_REPO
+        path.mkdir(parents=True, exist_ok=True)
+        pathname = path / CONFIG.git_repo.workdir.name
+        async with aiofiles.open(str(pathname.with_suffix(".mmd")), mode="w", encoding="utf-8") as writer:
+            content = "classDiagram\n"
+            logger.debug(content)
+            await writer.write(content)
+            # class names
+            rows = await graph_db.select(predicate=GraphKeyword.IS, object_=GraphKeyword.CLASS)
+            class_distinct = set()
+            relationship_distinct = set()
+            for r in rows:
+                await RebuildClassView._create_mermaid_class(r.subject, graph_db, writer, class_distinct)
+            for r in rows:
+                await RebuildClassView._create_mermaid_relationship(r.subject, graph_db, writer, relationship_distinct)
+
+    @staticmethod
+    async def _create_mermaid_class(ns_class_name, graph_db, file_writer, distinct):
+        fields = split_namespace(ns_class_name)
+        if len(fields) > 2:
+            # Ignore sub-class
+            return
+
+        class_view = ClassView(name=fields[1])
+        rows = await graph_db.select(subject=ns_class_name)
+        for r in rows:
+            name = split_namespace(r.object_)[-1]
+            name, visibility, abstraction = RebuildClassView._parse_name(name=name, language="python")
+            if r.predicate == GraphKeyword.HAS_CLASS_PROPERTY:
+                var_type = await RebuildClassView._parse_variable_type(r.object_, graph_db)
+                attribute = ClassAttribute(
+                    name=name, visibility=visibility, abstraction=bool(abstraction), value_type=var_type
+                )
+                class_view.attributes.append(attribute)
+            elif r.predicate == GraphKeyword.HAS_CLASS_FUNCTION:
+                method = ClassMethod(name=name, visibility=visibility, abstraction=bool(abstraction))
+                await RebuildClassView._parse_function_args(method, r.object_, graph_db)
+                class_view.methods.append(method)
+
+        # update graph db
+        await graph_db.insert(ns_class_name, GraphKeyword.HAS_CLASS_VIEW, class_view.model_dump_json())
+
+        content = class_view.get_mermaid(align=1)
+        logger.debug(content)
+        await file_writer.write(content)
+        distinct.add(ns_class_name)
+
+    @staticmethod
+    async def _create_mermaid_relationship(ns_class_name, graph_db, file_writer, distinct):
+        s_fields = split_namespace(ns_class_name)
+        if len(s_fields) > 2:
+            # Ignore sub-class
+            return
+
+        predicates = {GraphKeyword.IS + v + GraphKeyword.OF: v for v in [GENERALIZATION, COMPOSITION, AGGREGATION]}
+        mappings = {
+            GENERALIZATION: " <|-- ",
+            COMPOSITION: " *-- ",
+            AGGREGATION: " o-- ",
+        }
+        content = ""
+        for p, v in predicates.items():
+            rows = await graph_db.select(subject=ns_class_name, predicate=p)
+            for r in rows:
+                o_fields = split_namespace(r.object_)
+                if len(o_fields) > 2:
+                    # Ignore sub-class
+                    continue
+                relationship = mappings.get(v, " .. ")
+                link = f"{o_fields[1]}{relationship}{s_fields[1]}"
+                distinct.add(link)
+                content += f"\t{link}\n"
+
+        if content:
+            logger.debug(content)
+            await file_writer.write(content)
+
+    @staticmethod
+    def _parse_name(name: str, language="python"):
+        pattern = re.compile(r"<I>(.*?)<\/I>")
+        result = re.search(pattern, name)
+
+        abstraction = ""
+        if result:
+            name = result.group(1)
+            abstraction = "*"
+        if name.startswith("__"):
+            visibility = "-"
+        elif name.startswith("_"):
+            visibility = "#"
+        else:
+            visibility = "+"
+        return name, visibility, abstraction
+
+    @staticmethod
+    async def _parse_variable_type(ns_name, graph_db) -> str:
+        rows = await graph_db.select(subject=ns_name, predicate=GraphKeyword.HAS_TYPE_DESC)
+        if not rows:
+            return ""
+        vals = rows[0].object_.replace("'", "").split(":")
+        if len(vals) == 1:
+            return ""
+        val = vals[-1].strip()
+        return "" if val == "NoneType" else val + " "
+
+    @staticmethod
+    async def _parse_function_args(method: ClassMethod, ns_name: str, graph_db: GraphRepository):
+        rows = await graph_db.select(subject=ns_name, predicate=GraphKeyword.HAS_ARGS_DESC)
+        if not rows:
+            return
+        info = rows[0].object_.replace("'", "")
+
+        fs_tag = "("
+        ix = info.find(fs_tag)
+        fe_tag = "):"
+        eix = info.rfind(fe_tag)
+        if eix < 0:
+            fe_tag = ")"
+            eix = info.rfind(fe_tag)
+        args_info = info[ix + len(fs_tag) : eix].strip()
+        method.return_type = info[eix + len(fe_tag) :].strip()
+        if method.return_type == "None":
+            method.return_type = ""
+        if "(" in method.return_type:
+            method.return_type = method.return_type.replace("(", "Tuple[").replace(")", "]")
+
+        # parse args
+        if not args_info:
+            return
+        splitter_ixs = []
+        cost = 0
+        for i in range(len(args_info)):
+            if args_info[i] == "[":
+                cost += 1
+            elif args_info[i] == "]":
+                cost -= 1
+            if args_info[i] == "," and cost == 0:
+                splitter_ixs.append(i)
+        splitter_ixs.append(len(args_info))
+        args = []
+        ix = 0
+        for eix in splitter_ixs:
+            args.append(args_info[ix:eix])
+            ix = eix + 1
+        for arg in args:
+            parts = arg.strip().split(":")
+            if len(parts) == 1:
+                method.args.append(ClassAttribute(name=parts[0].strip()))
+                continue
+            method.args.append(ClassAttribute(name=parts[0].strip(), value_type=parts[-1].strip()))

metagpt/actions/write_code.py

@@ -130,7 +130,7 @@ async def run(self, *args, **kwargs) -> CodingContext:
         if not coding_context.code_doc:
             # avoid root_path pydantic ValidationError if use WriteCode alone
             root_path = CONFIG.src_workspace if CONFIG.src_workspace else ""
-            coding_context.code_doc = Document(filename=coding_context.filename, root_path=root_path)
+            coding_context.code_doc = Document(filename=coding_context.filename, root_path=str(root_path))
         coding_context.code_doc.content = code
         return coding_context
 

metagpt/const.py

@@ -126,3 +126,8 @@ def get_metagpt_root():
 
 # Message id
 IGNORED_MESSAGE_ID = "0"
+
+# Class Relationship
+GENERALIZATION = "Generalize"
+COMPOSITION = "Composite"
+AGGREGATION = "Aggregate"