diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.py index bd56cf2f579..347f58bfc3b 100644 --- a/autogen/agentchat/chat.py +++ b/autogen/agentchat/chat.py @@ -26,7 +26,9 @@ class ChatResult: summary: str = None """A summary obtained from the chat.""" cost: tuple = None # (dict, dict) - (total_cost, actual_cost_with_cache) - """The cost of the chat. a tuple of (total_cost, total_actual_cost), where total_cost is a dictionary of cost information, and total_actual_cost is a dictionary of information on the actual incurred cost with cache.""" + """The cost of the chat. a tuple of (total_cost, total_actual_cost), where total_cost is a + dictionary of cost information, and total_actual_cost is a dictionary of information on + the actual incurred cost with cache.""" human_input: List[str] = None """A list of human input solicited during the chat.""" @@ -141,25 +143,32 @@ def __post_carryover_processing(chat_info: Dict[str, Any]) -> None: def initiate_chats(chat_queue: List[Dict[str, Any]]) -> List[ChatResult]: """Initiate a list of chats. - Args: - chat_queue (List[Dict]): a list of dictionaries containing the information about the chats. - - Each dictionary should contain the input arguments for [`ConversableAgent.initiate_chat`](/docs/reference/agentchat/conversable_agent#initiate_chat). For example: - - "sender": the sender agent. - - "recipient": the recipient agent. - - "clear_history" (bool): whether to clear the chat history with the agent. Default is True. - - "silent" (bool or None): (Experimental) whether to print the messages in this conversation. Default is False. - - "cache" (AbstractCache or None): the cache client to use for this conversation. Default is None. - - "max_turns" (int or None): maximum number of turns for the chat. If None, the chat will continue until a termination condition is met. Default is None. - - "summary_method" (str or callable): a string or callable specifying the method to get a summary from the chat. Default is DEFAULT_summary_method, i.e., "last_msg". - - "summary_args" (dict): a dictionary of arguments to be passed to the summary_method. Default is {}. - - "message" (str, callable or None): if None, input() will be called to get the initial message. - - **context: additional context information to be passed to the chat. - - "carryover": It can be used to specify the carryover information to be passed to this chat. - If provided, we will combine this carryover with the "message" content when generating the initial chat - message in `generate_init_message`. - + chat_queue (List[Dict]): A list of dictionaries containing the information about the chats. + + Each dictionary should contain the input arguments for + [`ConversableAgent.initiate_chat`](/docs/reference/agentchat/conversable_agent#initiate_chat). + For example: + - `"sender"` - the sender agent. + - `"recipient"` - the recipient agent. + - `"clear_history" (bool) - whether to clear the chat history with the agent. + Default is True. + - `"silent"` (bool or None) - (Experimental) whether to print the messages in this + conversation. Default is False. + - `"cache"` (Cache or None) - the cache client to use for this conversation. + Default is None. + - `"max_turns"` (int or None) - maximum number of turns for the chat. If None, the chat + will continue until a termination condition is met. Default is None. + - `"summary_method"` (str or callable) - a string or callable specifying the method to get + a summary from the chat. Default is DEFAULT_summary_method, i.e., "last_msg". + - `"summary_args"` (dict) - a dictionary of arguments to be passed to the summary_method. + Default is {}. + - `"message"` (str, callable or None) - if None, input() will be called to get the + initial message. + - `**context` - additional context information to be passed to the chat. + - `"carryover"` - It can be used to specify the carryover information to be passed + to this chat. If provided, we will combine this carryover with the "message" content when + generating the initial chat message in `generate_init_message`. Returns: (list): a list of ChatResult objects corresponding to the finished chats in the chat_queue. """ @@ -228,11 +237,11 @@ async def a_initiate_chats(chat_queue: List[Dict[str, Any]]) -> Dict[int, ChatRe """(async) Initiate a list of chats. args: - Please refer to `initiate_chats`. + - Please refer to `initiate_chats`. returns: - (Dict): a dict of ChatId: ChatResult corresponding to the finished chats in the chat_queue. + - (Dict): a dict of ChatId: ChatResult corresponding to the finished chats in the chat_queue. """ consolidate_chat_info(chat_queue) _validate_recipients(chat_queue) diff --git a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py index f252f60e5ec..1d029a5192c 100644 --- a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py +++ b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py @@ -62,6 +62,10 @@ class RetrieveUserProxyAgent(UserProxyAgent): + """(In preview) The Retrieval-Augmented User Proxy retrieves document chunks based on the embedding + similarity, and sends them along with the question to the Retrieval-Augmented Assistant + """ + def __init__( self, name="RetrieveChatAgent", # default set to RetrieveChatAgent @@ -73,67 +77,106 @@ def __init__( r""" Args: name (str): name of the agent. + human_input_mode (str): whether to ask for human inputs every time a message is received. Possible values are "ALWAYS", "TERMINATE", "NEVER". 1. When "ALWAYS", the agent prompts for human input every time a message is received. Under this mode, the conversation stops when the human input is "exit", or when is_termination_msg is True and there is no human input. - 2. When "TERMINATE", the agent only prompts for human input only when a termination message is received or - the number of auto reply reaches the max_consecutive_auto_reply. - 3. When "NEVER", the agent will never prompt for human input. Under this mode, the conversation stops - when the number of auto reply reaches the max_consecutive_auto_reply or when is_termination_msg is True. + 2. When "TERMINATE", the agent only prompts for human input only when a termination + message is received or the number of auto reply reaches + the max_consecutive_auto_reply. + 3. When "NEVER", the agent will never prompt for human input. Under this mode, the + conversation stops when the number of auto reply reaches the + max_consecutive_auto_reply or when is_termination_msg is True. + is_termination_msg (function): a function that takes a message in the form of a dictionary and returns a boolean value indicating if this received message is a termination message. The dict can contain the following keys: "content", "role", "name", "function_call". + retrieve_config (dict or None): config for the retrieve agent. - To use default config, set to None. Otherwise, set to a dictionary with the following keys: - - task (Optional, str): the task of the retrieve chat. Possible values are "code", "qa" and "default". System - prompt will be different for different tasks. The default value is `default`, which supports both code and qa. - - client (Optional, chromadb.Client): the chromadb client. If key not provided, a default client `chromadb.Client()` - will be used. If you want to use other vector db, extend this class and override the `retrieve_docs` function. - - docs_path (Optional, Union[str, List[str]]): the path to the docs directory. It can also be the path to a single file, - the url to a single file or a list of directories, files and urls. Default is None, which works only if the collection is already created. - - extra_docs (Optional, bool): when true, allows adding documents with unique IDs without overwriting existing ones; when false, it replaces existing documents using default IDs, risking collection overwrite., - when set to true it enables the system to assign unique IDs starting from "length+i" for new document chunks, preventing the replacement of existing documents and facilitating the addition of more content to the collection.. - By default, "extra_docs" is set to false, starting document IDs from zero. This poses a risk as new documents might overwrite existing ones, potentially causing unintended loss or alteration of data in the collection. - - collection_name (Optional, str): the name of the collection. + + To use default config, set to None. Otherwise, set to a dictionary with the + following keys: + - `task` (Optional, str) - the task of the retrieve chat. Possible values are + "code", "qa" and "default". System prompt will be different for different tasks. + The default value is `default`, which supports both code and qa. + - `client` (Optional, chromadb.Client) - the chromadb client. If key not provided, a + default client `chromadb.Client()` will be used. If you want to use other + vector db, extend this class and override the `retrieve_docs` function. + - `docs_path` (Optional, Union[str, List[str]]) - the path to the docs directory. It + can also be the path to a single file, the url to a single file or a list + of directories, files and urls. Default is None, which works only if the + collection is already created. + - `extra_docs` (Optional, bool) - when true, allows adding documents with unique IDs + without overwriting existing ones; when false, it replaces existing documents + using default IDs, risking collection overwrite., when set to true it enables + the system to assign unique IDs starting from "length+i" for new document + chunks, preventing the replacement of existing documents and facilitating the + addition of more content to the collection.. + By default, "extra_docs" is set to false, starting document IDs from zero. + This poses a risk as new documents might overwrite existing ones, potentially + causing unintended loss or alteration of data in the collection. + - `collection_name` (Optional, str) - the name of the collection. If key not provided, a default name `autogen-docs` will be used. - - model (Optional, str): the model to use for the retrieve chat. + - `model` (Optional, str) - the model to use for the retrieve chat. If key not provided, a default model `gpt-4` will be used. - - chunk_token_size (Optional, int): the chunk token size for the retrieve chat. + - `chunk_token_size` (Optional, int) - the chunk token size for the retrieve chat. If key not provided, a default size `max_tokens * 0.4` will be used. - - context_max_tokens (Optional, int): the context max token size for the retrieve chat. + - `context_max_tokens` (Optional, int) - the context max token size for the + retrieve chat. If key not provided, a default size `max_tokens * 0.8` will be used. - - chunk_mode (Optional, str): the chunk mode for the retrieve chat. Possible values are - "multi_lines" and "one_line". If key not provided, a default mode `multi_lines` will be used. - - must_break_at_empty_line (Optional, bool): chunk will only break at empty line if True. Default is True. + - `chunk_mode` (Optional, str) - the chunk mode for the retrieve chat. Possible values + are "multi_lines" and "one_line". If key not provided, a default mode + `multi_lines` will be used. + - `must_break_at_empty_line` (Optional, bool) - chunk will only break at empty line + if True. Default is True. If chunk_mode is "one_line", this parameter will be ignored. - - embedding_model (Optional, str): the embedding model to use for the retrieve chat. - If key not provided, a default model `all-MiniLM-L6-v2` will be used. All available models - can be found at `https://www.sbert.net/docs/pretrained_models.html`. The default model is a - fast model. If you want to use a high performance model, `all-mpnet-base-v2` is recommended. - - embedding_function (Optional, Callable): the embedding function for creating the vector db. Default is None, - SentenceTransformer with the given `embedding_model` will be used. If you want to use OpenAI, Cohere, HuggingFace or - other embedding functions, you can pass it here, follow the examples in `https://docs.trychroma.com/embeddings`. - - customized_prompt (Optional, str): the customized prompt for the retrieve chat. Default is None. - - customized_answer_prefix (Optional, str): the customized answer prefix for the retrieve chat. Default is "". - If not "" and the customized_answer_prefix is not in the answer, `Update Context` will be triggered. - - update_context (Optional, bool): if False, will not apply `Update Context` for interactive retrieval. Default is True. - - get_or_create (Optional, bool): if True, will create/return a collection for the retrieve chat. This is the same as that used in chromadb. - Default is False. Will raise ValueError if the collection already exists and get_or_create is False. Will be set to True if docs_path is None. - - custom_token_count_function (Optional, Callable): a custom function to count the number of tokens in a string. - The function should take (text:str, model:str) as input and return the token_count(int). the retrieve_config["model"] will be passed in the function. - Default is autogen.token_count_utils.count_token that uses tiktoken, which may not be accurate for non-OpenAI models. - - custom_text_split_function (Optional, Callable): a custom function to split a string into a list of strings. - Default is None, will use the default function in `autogen.retrieve_utils.split_text_to_chunks`. - - custom_text_types (Optional, List[str]): a list of file types to be processed. Default is `autogen.retrieve_utils.TEXT_FORMATS`. - This only applies to files under the directories in `docs_path`. Explicitly included files and urls will be chunked regardless of their types. - - recursive (Optional, bool): whether to search documents recursively in the docs_path. Default is True. + - `embedding_model` (Optional, str) - the embedding model to use for the retrieve chat. + If key not provided, a default model `all-MiniLM-L6-v2` will be used. All available + models can be found at `https://www.sbert.net/docs/pretrained_models.html`. + The default model is a fast model. If you want to use a high performance model, + `all-mpnet-base-v2` is recommended. + - `embedding_function` (Optional, Callable) - the embedding function for creating the + vector db. Default is None, SentenceTransformer with the given `embedding_model` + will be used. If you want to use OpenAI, Cohere, HuggingFace or other embedding + functions, you can pass it here, + follow the examples in `https://docs.trychroma.com/embeddings`. + - `customized_prompt` (Optional, str) - the customized prompt for the retrieve chat. + Default is None. + - `customized_answer_prefix` (Optional, str) - the customized answer prefix for the + retrieve chat. Default is "". + If not "" and the customized_answer_prefix is not in the answer, + `Update Context` will be triggered. + - `update_context` (Optional, bool) - if False, will not apply `Update Context` for + interactive retrieval. Default is True. + - `get_or_create` (Optional, bool) - if True, will create/return a collection for the + retrieve chat. This is the same as that used in chromadb. + Default is False. Will raise ValueError if the collection already exists and + get_or_create is False. Will be set to True if docs_path is None. + - `custom_token_count_function` (Optional, Callable) - a custom function to count the + number of tokens in a string. + The function should take (text:str, model:str) as input and return the + token_count(int). the retrieve_config["model"] will be passed in the function. + Default is autogen.token_count_utils.count_token that uses tiktoken, which may + not be accurate for non-OpenAI models. + - `custom_text_split_function` (Optional, Callable) - a custom function to split a + string into a list of strings. + Default is None, will use the default function in + `autogen.retrieve_utils.split_text_to_chunks`. + - `custom_text_types` (Optional, List[str]) - a list of file types to be processed. + Default is `autogen.retrieve_utils.TEXT_FORMATS`. + This only applies to files under the directories in `docs_path`. Explicitly + included files and urls will be chunked regardless of their types. + - `recursive` (Optional, bool) - whether to search documents recursively in the + docs_path. Default is True. + `**kwargs` (dict): other kwargs in [UserProxyAgent](../user_proxy_agent#__init__). Example: - Example of overriding retrieve_docs - If you have set up a customized vector db, and it's not compatible with chromadb, you can easily plug in it with below code. + Example of overriding retrieve_docs - If you have set up a customized vector db, and it's + not compatible with chromadb, you can easily plug in it with below code. ```python class MyRetrieveUserProxyAgent(RetrieveUserProxyAgent): def query_vector_db( @@ -416,9 +459,9 @@ def message_generator(sender, recipient, context): sender (Agent): the sender agent. It should be the instance of RetrieveUserProxyAgent. recipient (Agent): the recipient agent. Usually it's the assistant agent. context (dict): the context for the message generation. It should contain the following keys: - - problem (str): the problem to be solved. - - n_results (int): the number of results to be retrieved. Default is 20. - - search_string (str): only docs that contain an exact match of this string will be retrieved. Default is "". + - `problem` (str) - the problem to be solved. + - `n_results` (int) - the number of results to be retrieved. Default is 20. + - `search_string` (str) - only docs that contain an exact match of this string will be retrieved. Default is "". Returns: str: the generated message ready to be sent to the recipient agent. """ diff --git a/notebook/agentchat_webscraping_with_apify.ipynb b/notebook/agentchat_webscraping_with_apify.ipynb new file mode 100644 index 00000000000..ad80f0d960c --- /dev/null +++ b/notebook/agentchat_webscraping_with_apify.ipynb @@ -0,0 +1,389 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Web Scraping using Apify Tools\n", + "\n", + "This notebook shows how to use Apify tools with AutoGen agents to\n", + "scrape data from a website and formate the output." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we need to install the Apify SDK and the AutoGen library." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "! pip install -qqq pyautogen apify-client" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting up the LLM configuration and the Apify API key is also required." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "config_list = [\n", + " {\"model\": \"gpt-4\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")},\n", + "]\n", + "\n", + "apify_api_key = os.getenv(\"APIFY_API_KEY\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's define the tool for scraping data from the website using Apify actor.\n", + "Read more about tool use in this [tutorial chapter](/docs/tutorial/tool-use)." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from typing_extensions import Annotated\n", + "from apify_client import ApifyClient\n", + "\n", + "\n", + "def scrape_page(url: Annotated[str, \"The URL of the web page to scrape\"]) -> Annotated[str, \"Scraped content\"]:\n", + " # Initialize the ApifyClient with your API token\n", + " client = ApifyClient(token=apify_api_key)\n", + "\n", + " # Prepare the Actor input\n", + " run_input = {\n", + " \"startUrls\": [{\"url\": url}],\n", + " \"useSitemaps\": False,\n", + " \"crawlerType\": \"playwright:firefox\",\n", + " \"includeUrlGlobs\": [],\n", + " \"excludeUrlGlobs\": [],\n", + " \"ignoreCanonicalUrl\": False,\n", + " \"maxCrawlDepth\": 0,\n", + " \"maxCrawlPages\": 1,\n", + " \"initialConcurrency\": 0,\n", + " \"maxConcurrency\": 200,\n", + " \"initialCookies\": [],\n", + " \"proxyConfiguration\": {\"useApifyProxy\": True},\n", + " \"maxSessionRotations\": 10,\n", + " \"maxRequestRetries\": 5,\n", + " \"requestTimeoutSecs\": 60,\n", + " \"dynamicContentWaitSecs\": 10,\n", + " \"maxScrollHeightPixels\": 5000,\n", + " \"removeElementsCssSelector\": \"\"\"nav, footer, script, style, noscript, svg,\n", + " [role=\\\"alert\\\"],\n", + " [role=\\\"banner\\\"],\n", + " [role=\\\"dialog\\\"],\n", + " [role=\\\"alertdialog\\\"],\n", + " [role=\\\"region\\\"][aria-label*=\\\"skip\\\" i],\n", + " [aria-modal=\\\"true\\\"]\"\"\",\n", + " \"removeCookieWarnings\": True,\n", + " \"clickElementsCssSelector\": '[aria-expanded=\"false\"]',\n", + " \"htmlTransformer\": \"readableText\",\n", + " \"readableTextCharThreshold\": 100,\n", + " \"aggressivePrune\": False,\n", + " \"debugMode\": True,\n", + " \"debugLog\": True,\n", + " \"saveHtml\": True,\n", + " \"saveMarkdown\": True,\n", + " \"saveFiles\": False,\n", + " \"saveScreenshots\": False,\n", + " \"maxResults\": 9999999,\n", + " \"clientSideMinChangePercentage\": 15,\n", + " \"renderingTypeDetectionPercentage\": 10,\n", + " }\n", + "\n", + " # Run the Actor and wait for it to finish\n", + " run = client.actor(\"aYG0l9s7dbB7j3gbS\").call(run_input=run_input)\n", + "\n", + " # Fetch and print Actor results from the run's dataset (if there are any)\n", + " text_data = \"\"\n", + " for item in client.dataset(run[\"defaultDatasetId\"]).iterate_items():\n", + " text_data += item.get(\"text\", \"\") + \"\\n\"\n", + "\n", + " average_token = 0.75\n", + " max_tokens = 20000 # slightly less than max to be safe 32k\n", + " text_data = text_data[: int(average_token * max_tokens)]\n", + " return text_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the agents and register the tool." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen import ConversableAgent, register_function\n", + "\n", + "# Create web scrapper agent.\n", + "scraper_agent = ConversableAgent(\n", + " \"WebScraper\",\n", + " llm_config={\"config_list\": config_list},\n", + " system_message=\"You are a web scrapper and you can scrape any web page using the tools provided. \"\n", + " \"Returns 'TERMINATE' when the scraping is done.\",\n", + ")\n", + "\n", + "# Create user proxy agent.\n", + "user_proxy_agent = ConversableAgent(\n", + " \"UserProxy\",\n", + " llm_config=False, # No LLM for this agent.\n", + " human_input_mode=\"NEVER\",\n", + " code_execution_config=False, # No code execution for this agent.\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") is not None and \"terminate\" in x[\"content\"].lower(),\n", + " default_auto_reply=\"Please continue if not finished, otherwise return 'TERMINATE'.\",\n", + ")\n", + "\n", + "# Register the function with the agents.\n", + "register_function(\n", + " scrape_page,\n", + " caller=scraper_agent,\n", + " executor=user_proxy_agent,\n", + " name=\"scrape_page\",\n", + " description=\"Scrape a web page and return the content.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start the conversation for scraping web data. We used the\n", + "`reflection_with_llm` option for summary method\n", + "to perform the formatting of the output into a desired format.\n", + "The summary method is called after the conversation is completed\n", + "given the complete history of the conversation." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "Can you scrape agentops.ai for me?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebScraper\u001b[0m (to UserProxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_0qok2jvCxOfv7HOA0oxPWneM): scrape_page *****\u001b[0m\n", + "Arguments: \n", + "{\n", + "\"url\": \"https://www.agentops.ai\"\n", + "}\n", + "\u001b[32m****************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION scrape_page...\u001b[0m\n", + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_0qok2jvCxOfv7HOA0oxPWneM) *****\u001b[0m\n", + "START NOW\n", + "Take your business to the next level with our features \n", + "AI Agents Suck.\n", + "We're Fixing That. \n", + "Build compliant AI agents with observability, evals, and replay analytics. No more black boxes and prompt guessing.\n", + "New! Introducing AgentOps\n", + "Three Lines of Code. Unlimited Testing. \n", + "Instant Testing + Debugging = Compliant AI Agents That Work\n", + "5\n", + "# Beginning of program's code (i.e. main.py, __init__.py)\n", + "6\n", + "ao_client = agentops.Client()\n", + "9\n", + "# (optional: record specific functions)\n", + "10\n", + "@ao_client.record_action('sample function being record')\n", + "11\n", + "def sample_function(...):\n", + "15\n", + "ao_client.end_session('Success')\n", + "Prototype to Production\n", + "Generous free limits, upgrade only when you need it.\n", + "\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebScraper\u001b[0m (to UserProxy):\n", + "\n", + "Sure, here's the information from the website agentops.ai:\n", + "\n", + "- Their main value proposition is to fix bad AI Agents and replace black boxes and prompt guessing with compliant, observable AI agents that come with evals and replay analytics.\n", + "- Their latest product is AgentOps. The simple and instant testing & debugging offered promises better-performing compliant AI agents.\n", + "- Integration is easy with just three lines of code.\n", + "- They let you record specific functions.\n", + "- They provide generous free limits and you only need to upgrade when necessary.\n", + "\n", + "Here's a sample of their code:\n", + "```python\n", + "ao_client = agentops.Client()\n", + "\n", + "# optional: record specific functions\n", + "@ao_client.record_action('sample function being record')\n", + "def sample_function(...):\n", + " ...\n", + "\n", + "ao_client.end_session('Success')\n", + "```\n", + "This code is for sample usage of their libraries/functions.\n", + "\n", + "Let me know if you need more specific details.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "Please continue if not finished, otherwise return 'TERMINATE'.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebScraper\u001b[0m (to UserProxy):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "chat_result = user_proxy_agent.initiate_chat(\n", + " scraper_agent,\n", + " message=\"Can you scrape agentops.ai for me?\",\n", + " summary_method=\"reflection_with_llm\",\n", + " summary_args={\n", + " \"summary_prompt\": \"\"\"Summarize the scraped content and format summary EXACTLY as follows:\n", + "---\n", + "*Company name*:\n", + "`Acme Corp`\n", + "---\n", + "*Website*:\n", + "`acmecorp.com`\n", + "---\n", + "*Description*:\n", + "`Company that does things.`\n", + "---\n", + "*Tags*:\n", + "`Manufacturing. Retail. E-commerce.`\n", + "---\n", + "*Takeaways*:\n", + "`Provides shareholders with value by selling products.`\n", + "---\n", + "*Questions*:\n", + "`What products do they sell? How do they make money? What is their market share?`\n", + "---\n", + "\"\"\"\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output is stored in the summary." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "---\n", + "*Company name*:\n", + "`AgentOps`\n", + "---\n", + "*Website*:\n", + "`agentops.ai`\n", + "---\n", + "*Description*:\n", + "`Company that aims to improve AI agents. They offer observed and evaluable AI agents with replay analytics as an alternative to black box models and blind prompting.`\n", + "---\n", + "*Tags*:\n", + "`Artificial Intelligence, AI agents, Observability, Analytics.`\n", + "---\n", + "*Takeaways*:\n", + "`Their product, AgentOps, allows for easy and instant testing and debugging of AI agents. Integration is as simple as writing three lines of code. They also provide generous free limits and mandate upgrades only when necessary.`\n", + "---\n", + "*Questions*:\n", + "`What differentiates AgentOps from other, similar products? How does their pricing scale with usage? What are the details of their \"generous free limits\"?`\n", + "---\n" + ] + } + ], + "source": [ + "print(chat_result.summary)" + ] + } + ], + "metadata": { + "front_matter": { + "description": "Scrapping web pages and summarizing the content using agents with tools.", + "tags": [ + "web scraping", + "apify", + "tool use" + ], + "title": "Web Scraper Agent using Apify Tools" + }, + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test/agentchat/contrib/capabilities/test_vision_capability.py b/test/agentchat/contrib/capabilities/test_vision_capability.py index a62d5245057..1266cd0a026 100644 --- a/test/agentchat/contrib/capabilities/test_vision_capability.py +++ b/test/agentchat/contrib/capabilities/test_vision_capability.py @@ -1,3 +1,4 @@ +import os from unittest.mock import MagicMock, patch import pytest @@ -5,6 +6,8 @@ from autogen.agentchat.conversable_agent import ConversableAgent try: + from PIL import Image + from autogen.agentchat.contrib.capabilities.vision_capability import VisionCapability except ImportError: skip_test = True @@ -21,6 +24,9 @@ def lmm_config(): } +img_name = os.path.abspath("test/test_files/test_image.png") + + @pytest.fixture def vision_capability(lmm_config): return VisionCapability(lmm_config, custom_caption_func=None) @@ -72,9 +78,9 @@ def test_process_last_received_message_text(mock_lmm_client, vision_capability): def test_process_last_received_message_with_image( mock_get_caption, mock_convert_base64, mock_get_image_data, vision_capability ): - content = [{"type": "image_url", "image_url": {"url": "notebook/viz_gc.png"}}] + content = [{"type": "image_url", "image_url": {"url": (img_name)}}] expected_caption = ( - " in case you can not see, the caption of this image is: A sample image caption.\n" + f" in case you can not see, the caption of this image is: A sample image caption.\n" ) processed_content = vision_capability.process_last_received_message(content) assert processed_content == expected_caption @@ -101,7 +107,7 @@ def caption_func(image_url: str, image_data=None, lmm_client=None) -> str: class TestCustomCaptionFunc: def test_custom_caption_func_with_valid_url(self, custom_caption_func): """Test custom caption function with a valid image URL.""" - image_url = "notebook/viz_gc.png" + image_url = img_name expected_caption = f"An image description. The image is from {image_url}." assert custom_caption_func(image_url) == expected_caption, "Caption does not match expected output." @@ -109,7 +115,7 @@ def test_process_last_received_message_with_custom_func(self, lmm_config, custom """Test processing a message containing an image URL with a custom caption function.""" vision_capability = VisionCapability(lmm_config, custom_caption_func=custom_caption_func) - image_url = "notebook/viz_gc.png" + image_url = img_name content = [{"type": "image_url", "image_url": {"url": image_url}}] expected_output = f" An image description. The image is from {image_url}." processed_content = vision_capability.process_last_received_message(content) diff --git a/test/test_files/test_image.png b/test/test_files/test_image.png new file mode 100644 index 00000000000..1ecece5e2f8 Binary files /dev/null and b/test/test_files/test_image.png differ diff --git a/website/docs/Contribute.md b/website/docs/Contribute.md deleted file mode 100644 index 92bea4897c8..00000000000 --- a/website/docs/Contribute.md +++ /dev/null @@ -1,269 +0,0 @@ -# Contributing - -This project welcomes and encourages all forms of contributions, including but not limited to: - -- Pushing patches. -- Code review of pull requests. -- Documentation, examples and test cases. -- Readability improvement, e.g., improvement on docstr and comments. -- Community participation in [issues](https://github.com/microsoft/autogen/issues), [discussions](https://github.com/microsoft/autogen/discussions), [discord](https://aka.ms/autogen-dc), and [twitter](https://twitter.com/pyautogen). -- Tutorials, blog posts, talks that promote the project. -- Sharing application scenarios and/or related research. - -Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit . - -If you are new to GitHub [here](https://help.github.com/categories/collaborating-with-issues-and-pull-requests/) is a detailed help source on getting involved with development on GitHub. - -When you submit a pull request, a CLA bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - -## Roadmaps - -To see what we are working on and what we plan to work on, please check our -[Roadmap Issues](https://aka.ms/autogen-roadmap). - -## How to make a good bug report - -When you submit an issue to [GitHub](https://github.com/microsoft/autogen/issues), please do your best to -follow these guidelines! This will make it a lot easier to provide you with good -feedback: - -- The ideal bug report contains a short reproducible code snippet. This way - anyone can try to reproduce the bug easily (see [this](https://stackoverflow.com/help/mcve) for more details). If your snippet is - longer than around 50 lines, please link to a [gist](https://gist.github.com) or a GitHub repo. - -- If an exception is raised, please **provide the full traceback**. - -- Please include your **operating system type and version number**, as well as - your **Python, autogen, scikit-learn versions**. The version of autogen - can be found by running the following code snippet: - -```python -import autogen -print(autogen.__version__) -``` - -- Please ensure all **code snippets and error messages are formatted in - appropriate code blocks**. See [Creating and highlighting code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks) - for more details. - -## Becoming a Reviewer - -There is currently no formal reviewer solicitation process. Current reviewers identify reviewers from active contributors. If you are willing to become a reviewer, you are welcome to let us know on discord. - -## Guidance for Maintainers - -### General - -- Be a member of the community and treat everyone as a member. Be inclusive. -- Help each other and encourage mutual help. -- Actively post and respond. -- Keep open communication. - -### Pull Requests - -- For new PR, decide whether to close without review. If not, find the right reviewers. The default reviewer is microsoft/autogen. Ask users who can benefit from the PR to review it. - -- For old PR, check the blocker: reviewer or PR creator. Try to unblock. Get additional help when needed. -- When requesting changes, make sure you can check back in time because it blocks merging. -- Make sure all the checks are passed. -- For changes that require running OpenAI tests, make sure the OpenAI tests pass too. Running these tests requires approval. -- In general, suggest small PRs instead of a giant PR. -- For documentation change, request snapshot of the compiled website, or compile by yourself to verify the format. -- For new contributors who have not signed the contributing agreement, remind them to sign before reviewing. -- For multiple PRs which may have conflict, coordinate them to figure out the right order. -- Pay special attention to: - - Breaking changes. Don’t make breaking changes unless necessary. Don’t merge to main until enough headsup is provided and a new release is ready. - - Test coverage decrease. - - Changes that may cause performance degradation. Do regression test when test suites are available. - - Discourage **change to the core library** when there is an alternative. - -### Issues and Discussions - -- For new issues, write a reply, apply a label if relevant. Ask on discord when necessary. For roadmap issues, add to the roadmap project and encourage community discussion. Mention relevant experts when necessary. - -- For old issues, provide an update or close. Ask on discord when necessary. Encourage PR creation when relevant. -- Use “good first issue” for easy fix suitable for first-time contributors. -- Use “task list” for issues that require multiple PRs. -- For discussions, create an issue when relevant. Discuss on discord when appropriate. - -## Docker for Development - -For developers contributing to the AutoGen project, we offer a specialized Docker environment. This setup is designed to streamline the development process, ensuring that all contributors work within a consistent and well-equipped environment. - -### Autogen Developer Image (autogen_dev_img) - -- **Purpose**: The `autogen_dev_img` is tailored for contributors to the AutoGen project. It includes a suite of tools and configurations that aid in the development and testing of new features or fixes. -- **Usage**: This image is recommended for developers who intend to contribute code or documentation to AutoGen. -- **Forking the Project**: It's advisable to fork the AutoGen GitHub project to your own repository. This allows you to make changes in a separate environment without affecting the main project. -- **Updating Dockerfile**: Modify your copy of `Dockerfile` in the `dev` folder as needed for your development work. -- **Submitting Pull Requests**: Once your changes are ready, submit a pull request from your branch to the upstream AutoGen GitHub project for review and integration. For more details on contributing, see the [AutoGen Contributing](https://microsoft.github.io/autogen/docs/Contribute) page. - -### Building the Developer Docker Image - -- To build the developer Docker image (`autogen_dev_img`), use the following commands: - - ```bash - docker build -f .devcontainer/dev/Dockerfile -t autogen_dev_img https://github.com/microsoft/autogen.git#main - ``` - -- For building the developer image built from a specific Dockerfile in a branch other than main/master - - ```bash - # clone the branch you want to work out of - git clone --branch {branch-name} https://github.com/microsoft/autogen.git - - # cd to your new directory - cd autogen - - # build your Docker image - docker build -f .devcontainer/dev/Dockerfile -t autogen_dev-srv_img . - ``` - -### Using the Developer Docker Image - -Once you have built the `autogen_dev_img`, you can run it using the standard Docker commands. This will place you inside the containerized development environment where you can run tests, develop code, and ensure everything is functioning as expected before submitting your contributions. - -```bash -docker run -it -p 8081:3000 -v `pwd`/autogen-newcode:newstuff/ autogen_dev_img bash -``` - -- Note that the `pwd` is shorthand for present working directory. Thus, any path after the pwd is relative to that. If you want a more verbose method you could remove the "`pwd`/autogen-newcode" and replace it with the full path to your directory - -```bash -docker run -it -p 8081:3000 -v /home/AutoGenDeveloper/autogen-newcode:newstuff/ autogen_dev_img bash -``` - -### Develop in Remote Container - -If you use vscode, you can open the autogen folder in a [Container](https://code.visualstudio.com/docs/remote/containers). -We have provided the configuration in [devcontainer](https://github.com/microsoft/autogen/blob/main/.devcontainer). They can be used in GitHub codespace too. Developing AutoGen in dev containers is recommended. - -### Pre-commit - -Run `pre-commit install` to install pre-commit into your git hooks. Before you commit, run -`pre-commit run` to check if you meet the pre-commit requirements. If you use Windows (without WSL) and can't commit after installing pre-commit, you can run `pre-commit uninstall` to uninstall the hook. In WSL or Linux this is supposed to work. - -### Write tests - -Tests are automatically run via GitHub actions. There are two workflows: - -1. [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) -1. [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml) - -The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml). - -Make sure all tests pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) checks to pass - -#### Running tests locally - -To run tests, install the [test] option: - -```bash -pip install -e."[test]" -``` - -Then you can run the tests from the `test` folder using the following command: - -```bash -pytest test -``` - -Tests for the `autogen.agentchat.contrib` module may be skipped automatically if the -required dependencies are not installed. Please consult the documentation for -each contrib module to see what dependencies are required. - -See [here](https://github.com/microsoft/autogen/blob/main/notebook/contributing.md#testing) for how to run notebook tests. - -#### Skip flags for tests - -- `--skip-openai` for skipping tests that require access to OpenAI services. -- `--skip-docker` for skipping tests that explicitly use docker -- `--skip-redis` for skipping tests that require a Redis server - -For example, the following command will skip tests that require access to -OpenAI and docker services: - -```bash -pytest test --skip-openai --skip-docker -``` - -### Coverage - -Any code you commit should not decrease coverage. To run all unit tests, install the [test] option: - -```bash -pip install -e."[test]" -coverage run -m pytest test -``` - -Then you can see the coverage report by -`coverage report -m` or `coverage html`. - -### Documentation - -#### How to get a notebook rendered on the website - -See [here](https://github.com/microsoft/autogen/blob/main/notebook/contributing.md#how-to-get-a-notebook-displayed-on-the-website) for instructions on how to get a notebook in the `notebook` directory rendered on the website. - -#### Build documentation locally -1\. To build and test documentation locally, first install [Node.js](https://nodejs.org/en/download/). For example, -```bash -nvm install --lts -``` - -Then, install `yarn` and other required packages: -```bash -npm install --global yarn -pip install pydoc-markdown pyyaml termcolor -``` - -2\. You also need to install quarto. Please click on the `Pre-release` tab from [this website](https://quarto.org/docs/download/) to download the latest version of `quarto` and install it. Ensure that the `quarto` version is `1.5.23` or higher. - -3\. Finally, run the following commands to build: - -```console -cd website -yarn install --frozen-lockfile --ignore-engines -pydoc-markdown -python process_notebooks.py render -yarn start -``` - -The last command starts a local development server and opens up a browser window. -Most changes are reflected live without having to restart the server. - -#### Build with Docker -To build and test documentation within a docker container. Use the Dockerfile in the `dev` folder as described above to build your image: - -```bash -docker build -f .devcontainer/dev/Dockerfile -t autogen_dev_img https://github.com/microsoft/autogen.git#main -``` - -Then start the container like so, this will log you in and ensure that Docker port 3000 is mapped to port 8081 on your local machine - -```bash -docker run -it -p 8081:3000 -v `pwd`/autogen-newcode:newstuff/ autogen_dev_img bash -``` - -Once at the CLI in Docker run the following commands: - -```bash -cd website -yarn install --frozen-lockfile --ignore-engines -pydoc-markdown -python process_notebooks.py render -yarn start --host 0.0.0.0 --port 3000 -``` - -Once done you should be able to access the documentation at `http://127.0.0.1:8081/autogen` - -Note: -some tips in this guide are based off the contributor guide from [flaml](https://microsoft.github.io/FLAML/docs/Contribute). diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 155921719e8..3da637d85ee 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -54,6 +54,7 @@ Links to notebook examples: - Constrained Responses via Guidance - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_guidance.ipynb) - Browse the Web with Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_surfer.ipynb) - **SQL**: Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_sql_spider.ipynb) +- **Web Scraping**: Web Scraping with Apify - [View Notebook](/docs/notebooks/agentchat_webscraping_with_apify). ### Human Involvement diff --git a/website/docs/Getting-Started.mdx b/website/docs/Getting-Started.mdx index d1ed2a56a5e..0320c2b0bab 100644 --- a/website/docs/Getting-Started.mdx +++ b/website/docs/Getting-Started.mdx @@ -131,7 +131,7 @@ The figure below shows an example conversation flow with AutoGen. - Follow on [Twitter](https://twitter.com/pyautogen) - See our [roadmaps](https://aka.ms/autogen-roadmap) -If you like our project, please give it a [star](https://github.com/microsoft/autogen/stargazers) on GitHub. If you are interested in contributing, please read [Contributor's Guide](/docs/Contribute). +If you like our project, please give it a [star](https://github.com/microsoft/autogen/stargazers) on GitHub. If you are interested in contributing, please read [Contributor's Guide](/docs/contributor-guide/contributing).