Merge pull request #820 from shenchucheng/feature-remove-config

Feature remove global config in the search/browser engine

config/config2.yaml.example

@@ -11,6 +11,10 @@ search:
   api_key: "YOUR_API_KEY"
   cse_id: "YOUR_CSE_ID"
 
+browser:
+  engine: "playwright"  # playwright/selenium
+  browser_type: "chromium"  # playwright: chromium/firefox/webkit; selenium: chrome/firefox/edge/ie
+
 mermaid:
   engine: "pyppeteer"
   path: "/Applications/Google Chrome.app"

examples/search_with_specific_engine.py

@@ -5,17 +5,20 @@
 import asyncio
 
 from metagpt.roles import Searcher
-from metagpt.tools import SearchEngineType
+from metagpt.tools.search_engine import SearchEngine, SearchEngineType
 
 
 async def main():
     question = "What are the most interesting human facts?"
+    kwargs = {"api_key": "", "cse_id": "", "proxy": None}
     # Serper API
-    # await Searcher(engine=SearchEngineType.SERPER_GOOGLE).run(question)
+    # await Searcher(search_engine=SearchEngine(engine=SearchEngineType.SERPER_GOOGLE, **kwargs)).run(question)
     # SerpAPI
-    await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run(question)
+    # await Searcher(search_engine=SearchEngine(engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs)).run(question)
     # Google API
-    # await Searcher(engine=SearchEngineType.DIRECT_GOOGLE).run(question)
+    # await Searcher(search_engine=SearchEngine(engine=SearchEngineType.DIRECT_GOOGLE, **kwargs)).run(question)
+    # DDG API
+    await Searcher(search_engine=SearchEngine(engine=SearchEngineType.DUCK_DUCK_GO, **kwargs)).run(question)
 
 
 if __name__ == "__main__":

metagpt/actions/research.py

@@ -3,15 +3,15 @@
 from __future__ import annotations
 
 import asyncio
-from typing import Callable, Optional, Union
+from typing import Any, Callable, Optional, Union
 
-from pydantic import Field, parse_obj_as
+from pydantic import TypeAdapter, model_validator
 
 from metagpt.actions import Action
 from metagpt.config2 import config
 from metagpt.logs import logger
 from metagpt.tools.search_engine import SearchEngine
-from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType
+from metagpt.tools.web_browser_engine import WebBrowserEngine
 from metagpt.utils.common import OutputParser
 from metagpt.utils.text import generate_prompt_chunk, reduce_message_length
 
@@ -81,10 +81,16 @@ class CollectLinks(Action):
     name: str = "CollectLinks"
     i_context: Optional[str] = None
     desc: str = "Collect links from a search engine."
-
-    search_engine: SearchEngine = Field(default_factory=SearchEngine)
+    search_func: Optional[Any] = None
+    search_engine: Optional[SearchEngine] = None
     rank_func: Optional[Callable[[list[str]], None]] = None
 
+    @model_validator(mode="after")
+    def validate_engine_and_run_func(self):
+        if self.search_engine is None:
+            self.search_engine = SearchEngine.from_search_config(self.config.search, proxy=self.config.proxy)
+        return self
+
     async def run(
         self,
         topic: str,
@@ -107,7 +113,7 @@ async def run(
         keywords = await self._aask(SEARCH_TOPIC_PROMPT, [system_text])
         try:
             keywords = OutputParser.extract_struct(keywords, list)
-            keywords = parse_obj_as(list[str], keywords)
+            keywords = TypeAdapter(list[str]).validate_python(keywords)
         except Exception as e:
             logger.exception(f"fail to get keywords related to the research topic '{topic}' for {e}")
             keywords = [topic]
@@ -133,7 +139,7 @@ def gen_msg():
         queries = await self._aask(prompt, [system_text])
         try:
             queries = OutputParser.extract_struct(queries, list)
-            queries = parse_obj_as(list[str], queries)
+            queries = TypeAdapter(list[str]).validate_python(queries)
         except Exception as e:
             logger.exception(f"fail to break down the research question due to {e}")
             queries = keywords
@@ -178,15 +184,17 @@ class WebBrowseAndSummarize(Action):
     i_context: Optional[str] = None
     desc: str = "Explore the web and provide summaries of articles and webpages."
     browse_func: Union[Callable[[list[str]], None], None] = None
-    web_browser_engine: Optional[WebBrowserEngine] = WebBrowserEngineType.PLAYWRIGHT
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-        self.web_browser_engine = WebBrowserEngine(
-            engine=WebBrowserEngineType.CUSTOM if self.browse_func else WebBrowserEngineType.PLAYWRIGHT,
-            run_func=self.browse_func,
-        )
+    web_browser_engine: Optional[WebBrowserEngine] = None
+
+    @model_validator(mode="after")
+    def validate_engine_and_run_func(self):
+        if self.web_browser_engine is None:
+            self.web_browser_engine = WebBrowserEngine.from_browser_config(
+                self.config.browser,
+                browse_func=self.browse_func,
+                proxy=self.config.proxy,
+            )
+        return self
 
     async def run(
         self,

metagpt/actions/search_and_summarize.py

@@ -5,15 +5,14 @@
 @Author  : alexanderwu
 @File    : search_google.py
 """
-from typing import Any, Optional
+from typing import Optional
 
 import pydantic
 from pydantic import model_validator
 
 from metagpt.actions import Action
 from metagpt.logs import logger
 from metagpt.schema import Message
-from metagpt.tools import SearchEngineType
 from metagpt.tools.search_engine import SearchEngine
 
 SEARCH_AND_SUMMARIZE_SYSTEM = """### Requirements
@@ -105,21 +104,19 @@
 class SearchAndSummarize(Action):
     name: str = ""
     content: Optional[str] = None
-    engine: Optional[SearchEngineType] = None
-    search_func: Optional[Any] = None
     search_engine: SearchEngine = None
     result: str = ""
 
     @model_validator(mode="after")
-    def validate_engine_and_run_func(self):
-        if self.engine is None:
-            self.engine = self.config.search_engine
-        try:
-            search_engine = SearchEngine(engine=self.engine, run_func=self.search_func)
-        except pydantic.ValidationError:
-            search_engine = None
-
-        self.search_engine = search_engine
+    def validate_search_engine(self):
+        if self.search_engine is None:
+            try:
+                config = self.config
+                search_engine = SearchEngine.from_search_config(config.search, proxy=config.proxy)
+            except pydantic.ValidationError:
+                search_engine = None
+
+            self.search_engine = search_engine
         return self
 
     async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str:

metagpt/config2.py

@@ -51,7 +51,7 @@ class Config(CLIParams, YamlModel):
     proxy: str = ""
 
     # Tool Parameters
-    search: Optional[SearchConfig] = None
+    search: SearchConfig = SearchConfig()
     browser: BrowserConfig = BrowserConfig()
     mermaid: MermaidConfig = MermaidConfig()
 

metagpt/configs/browser_config.py

@@ -15,6 +15,6 @@ class BrowserConfig(YamlModel):
     """Config for Browser"""
 
     engine: WebBrowserEngineType = WebBrowserEngineType.PLAYWRIGHT
-    browser: Literal["chrome", "firefox", "edge", "ie"] = "chrome"
-    driver: Literal["chromium", "firefox", "webkit"] = "chromium"
-    path: str = ""
+    browser_type: Literal["chromium", "firefox", "webkit", "chrome", "firefox", "edge", "ie"] = "chromium"
+    """If the engine is Playwright, the value should be one of "chromium", "firefox", or "webkit". If it is Selenium, the value
+    should be either "chrome", "firefox", "edge", or "ie"."""

metagpt/configs/search_config.py

@@ -5,13 +5,16 @@
 @Author  : alexanderwu
 @File    : search_config.py
 """
+from typing import Callable, Optional
+
 from metagpt.tools import SearchEngineType
 from metagpt.utils.yaml_model import YamlModel
 
 
 class SearchConfig(YamlModel):
     """Config for Search"""
 
-    api_key: str
-    api_type: SearchEngineType = SearchEngineType.SERPAPI_GOOGLE
+    api_type: SearchEngineType = SearchEngineType.DUCK_DUCK_GO
+    api_key: str = ""
     cse_id: str = ""  # for google
+    search_func: Optional[Callable] = None

metagpt/context_mixin.py

@@ -7,7 +7,7 @@
 """
 from typing import Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from metagpt.config2 import Config
 from metagpt.context import Context
@@ -17,7 +17,7 @@
 class ContextMixin(BaseModel):
     """Mixin class for context and config"""
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
 
     # Pydantic has bug on _private_attr when using inheritance, so we use private_* instead
     # - https://github.com/pydantic/pydantic/issues/7142
@@ -32,15 +32,18 @@ class ContextMixin(BaseModel):
     # Env/Role/Action will use this llm as private llm, or use self.context._llm instance
     private_llm: Optional[BaseLLM] = Field(default=None, exclude=True)
 
-    def __init__(
+    @model_validator(mode="after")
+    def validate_extra(self):
+        self._process_extra(**(self.model_extra or {}))
+        return self
+
+    def _process_extra(
         self,
         context: Optional[Context] = None,
         config: Optional[Config] = None,
         llm: Optional[BaseLLM] = None,
-        **kwargs,
     ):
-        """Initialize with config"""
-        super().__init__(**kwargs)
+        """Process the extra field"""
         self.set_context(context)
         self.set_config(config)
         self.set_llm(llm)

metagpt/roles/sales.py

@@ -8,12 +8,12 @@
 
 from typing import Optional
 
-from pydantic import Field
+from pydantic import Field, model_validator
 
 from metagpt.actions import SearchAndSummarize, UserRequirement
 from metagpt.document_store.base_store import BaseStore
 from metagpt.roles import Role
-from metagpt.tools import SearchEngineType
+from metagpt.tools.search_engine import SearchEngine
 
 
 class Sales(Role):
@@ -29,14 +29,13 @@ class Sales(Role):
 
     store: Optional[BaseStore] = Field(default=None, exclude=True)
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._set_store(self.store)
-
-    def _set_store(self, store):
-        if store:
-            action = SearchAndSummarize(name="", engine=SearchEngineType.CUSTOM_ENGINE, search_func=store.asearch)
+    @model_validator(mode="after")
+    def validate_stroe(self):
+        if self.store:
+            search_engine = SearchEngine.from_search_func(search_func=self.store.asearch, proxy=self.config.proxy)
+            action = SearchAndSummarize(search_engine=search_engine, context=self.context)
         else:
-            action = SearchAndSummarize()
+            action = SearchAndSummarize
         self.set_actions([action])
         self._watch([UserRequirement])
+        return self

metagpt/roles/searcher.py

@@ -8,15 +8,17 @@
         the `cause_by` value in the `Message` to a string to support the new message distribution feature.
 """
 
-from pydantic import Field
+from typing import Optional
+
+from pydantic import Field, model_validator
 
 from metagpt.actions import SearchAndSummarize
 from metagpt.actions.action_node import ActionNode
 from metagpt.actions.action_output import ActionOutput
 from metagpt.logs import logger
 from metagpt.roles import Role
 from metagpt.schema import Message
-from metagpt.tools import SearchEngineType
+from metagpt.tools.search_engine import SearchEngine
 
 
 class Searcher(Role):
@@ -28,33 +30,22 @@ class Searcher(Role):
         profile (str): Role profile.
         goal (str): Goal of the searcher.
         constraints (str): Constraints or limitations for the searcher.
-        engine (SearchEngineType): The type of search engine to use.
+        search_engine (SearchEngine): The search engine to use.
     """
 
     name: str = Field(default="Alice")
     profile: str = Field(default="Smart Assistant")
     goal: str = "Provide search services for users"
     constraints: str = "Answer is rich and complete"
-    engine: SearchEngineType = SearchEngineType.SERPAPI_GOOGLE
-
-    def __init__(self, **kwargs) -> None:
-        """
-        Initializes the Searcher role with given attributes.
+    search_engine: Optional[SearchEngine] = None
 
-        Args:
-            name (str): Name of the searcher.
-            profile (str): Role profile.
-            goal (str): Goal of the searcher.
-            constraints (str): Constraints or limitations for the searcher.
-            engine (SearchEngineType): The type of search engine to use.
-        """
-        super().__init__(**kwargs)
-        self.set_actions([SearchAndSummarize(engine=self.engine)])
-
-    def set_search_func(self, search_func):
-        """Sets a custom search function for the searcher."""
-        action = SearchAndSummarize(name="", engine=SearchEngineType.CUSTOM_ENGINE, search_func=search_func)
-        self.set_actions([action])
+    @model_validator(mode="after")
+    def post_root(self):
+        if self.search_engine:
+            self.set_actions([SearchAndSummarize(search_engine=self.search_engine, context=self.context)])
+        else:
+            self.set_actions([SearchAndSummarize])
+        return self
 
     async def _act_sp(self) -> Message:
         """Performs the search action in a single process."""