From bc9a87e4a7cf3b0b0c9ec7debe4d8696acb43e24 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Thu, 26 Oct 2023 20:53:55 +0900 Subject: [PATCH 01/74] init notebook for agent set up --- ASSISTANT_CONFIG_LIST | 251 ++++++++++++++++++++++++++++++++ notebook/oai_create_agent.ipynb | 124 ++++++++++++++++ 2 files changed, 375 insertions(+) create mode 100644 ASSISTANT_CONFIG_LIST create mode 100644 notebook/oai_create_agent.ipynb diff --git a/ASSISTANT_CONFIG_LIST b/ASSISTANT_CONFIG_LIST new file mode 100644 index 00000000000..1d7a5c66119 --- /dev/null +++ b/ASSISTANT_CONFIG_LIST @@ -0,0 +1,251 @@ +{ + "gpt-4": { + "base": "GPT", + "type": ["chat", "instruct", "code"], + "language": ["en", "cn", "de", "es", "fr", "jp", "kr"], + "params": "online", + "context": "8192", + "release": "2023", + "benchmark": {}, + "description": "A set of models that improve on GPT-3.5 and can understand as well as generate natural language or code." + }, + "gpt-3.5-turbo": { + "base": "GPT", + "type": ["chat", "instruct", "code"], + "language": ["en", "cn", "de", "es", "fr", "jp", "kr"], + "params": "online", + "context": "4096", + "release": "2023", + "benchmark": {}, + "description": "A set of models that improve on GPT-3 and can understand as well as generate natural language or code." + }, + "meta-llama/Llama-2-7b-chat-hf": { + "base": "LLAMA", + "type": ["chat", "instruct", "code"], + "language": ["en"], + "params": "7b", + "context": "4096", + "precision": "fp16", + "release": "2023", + "benchmark": { + "ARC": 52.9, + "HellaSwag": 78.55, + "MMLU": 48.32, + "TruthfulQA": 45.57, + "avg": 56.34 + }, + "description": "Llama 2 is a collection of pretrained and fine-tuned generative text models ranging in scale from 7 billion to 70 billion parameters. This is the description for the 7B fine-tuned model, optimized for dialogue use cases and converted for the Hugging Face Transformers format." + }, + "meta-llama/Llama-2-13b-chat-hf": { + "base": "LLAMA", + "type": ["chat", "instruct", "code"], + "language": ["en"], + "params": "13b", + "context": "4096", + "precision": "fp16", + "release": "2023", + "benchmark": { + "ARC": 59.04, + "HellaSwag": 81.94, + "MMLU": 54.64, + "TruthfulQA": 44.12, + "avg": 59.93 + }, + "description": "Llama 2 is a collection of pretrained and fine-tuned generative text models ranging in scale from 7 billion to 70 billion parameters. This is the description for the 13B fine-tuned model, optimized for dialogue use cases and converted for the Hugging Face Transformers format." + }, + "meta-llama/Llama-2-70b-chat-hf": { + "model_name": "LLAMA", + "type": ["chat", "instruct", "code"], + "language": ["en"], + "params": "70b", + "context": "4096", + "precision": "fp16", + "release": "2023", + "benchmark": { + "ARC": 64.59, + "HellaSwag": 85.88, + "MMLU": 63.91, + "TruthfulQA": 52.8, + "avg": 66.8 + }, + "description": "Llama 2 is a collection of pretrained and fine-tuned generative text models ranging in scale from 7 billion to 70 billion parameters. This is the description for the 70B fine-tuned model, optimized for dialogue use cases and converted for the Hugging Face Transformers format." + }, + "tiiuae/falcon-7b-instruct": { + "base": "Falcon", + "type": ["chat", "instruct", "code"], + "language": ["en", "fr"], + "params": "7b", + "context": "2048", + "precision": "fp16", + "release": "2023", + "benchmark": {}, + "description": "Falcon-7B-Instruct is a 7B parameters causal decoder-only model built by TII based on Falcon-7B and finetuned on a mixture of chat/instruct datasets." + }, + "tiiuae/falcon-40b-instruct": { + "base": "Falcon", + "type": ["chat", "instruct", "code"], + "language": ["en", "fr"], + "params": "40b", + "context": "2048", + "precision": "fp16", + "release": "2023", + "benchmark": { + "ARC": 61.6, + "HellaSwag": 84.31, + "MMLU": 55.45, + "TruthfulQA": 52.52, + "avg": 63.47 + }, + "description": "Falcon-40B-Instruct is a 40B parameters causal decoder-only model built by TII based on Falcon-40B and finetuned on a mixture of Baize." + }, + "tiiuae/falcon-180B-chat": { + "base": "Falcon", + "type": ["chat", "instruct", "code"], + "language": ["en", "de", "es", "fr"], + "params": "180b", + "context": "2048", + "precision": "fp16", + "release": "2023", + "benchmark": {}, + "description": "Falcon-180B-Chat is a 180B parameters causal decoder-only model built by TII based on Falcon-180B and finetuned on a mixture of Ultrachat, Platypus and Airoboros." + }, + "FlagAlpha/Llama2-Chinese-7b-Chat": { + "base": "LLAMA", + "type": ["chat", "instruct", "code"], + "language": ["cn", "en"], + "params": "7b", + "context": "4096", + "precision": "fp16", + "release": "2023", + "benchmark": { + "ARC": 52.39, + "HellaSwag": 77.52, + "MMLU": 47.72, + "TruthfulQA": 46.87, + "avg": 56.13 + }, + "description": "Since Llama2 itself has weak Chinese alignment, we use the Chinese instruction set and fine-tune for meta-llama/Llama-2-7b-chat-hf to make it capable of strong Chinese dialog." + }, + "FlagAlpha/Llama2-Chinese-13b-Chat": { + "base": "LLAMA", + "type": ["chat", "instruct", "code"], + "language": ["cn", "en"], + "params": "13b", + "context": "4096", + "precision": "fp16", + "release": "2023", + "benchmark": { + "ARC": 55.97, + "HellaSwag": 82.05, + "MMLU": 54.74, + "TruthfulQA": 48.9, + "avg": 60.41 + }, + "description": "Since Llama2 itself has weak Chinese alignment, we use the Chinese instruction set and fine-tune for meta-llama/Llama-2-13b-chat-hf to make it capable of strong Chinese dialog." + }, + "baichuan-inc/Baichuan2-7B-Chat": { + "base": "Baichuan", + "type": ["chat", "instruct", "code"], + "language": ["cn", "en"], + "params": "7b", + "context": "4096", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Baichuan 2 (7b) is the new generation of large-scale open-source language models launched by Baichuan Intelligence inc.. It is trained on a high-quality corpus with 2.6 trillion tokens and has achieved the promising performance in authoritative Chinese and English benchmarks of the same size." + }, + "baichuan-inc/Baichuan2-13B-Chat": { + "base": "Baichuan", + "type": ["chat", "instruct", "code"], + "language": ["cn", "en"], + "params": "13b", + "context": "4096", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Baichuan 2 (13b) is the new generation of large-scale open-source language models launched by Baichuan Intelligence inc.. It is trained on a high-quality corpus with 2.6 trillion tokens and has achieved the promising performance in authoritative Chinese and English benchmarks of the same size." + }, + "elyza/ELYZA-japanese-Llama-2-7b-instruct": { + "base": "LLAMA", + "type": ["instruct", "code"], + "language": ["jp", "en"], + "params": "7b", + "context": "4096", + "precision": "fp16", + "release": "2023", + "benchmark": { + "ARC": 53.16, + "HellaSwag": 78.25, + "MMLU": 47.07, + "TruthfulQA": 39.08, + "avg": 54.39 + }, + "description": "ELYZA-japanese-Llama-2-7b is a model based on Llama2 with additional find-tuning to extend Japanese language capabilities." + }, + "codellama/CodeLlama-7b-Instruct-hf": { + "base": "LLAMA", + "type": ["code", "instruct"], + "language": ["en"], + "params": "34b", + "context": "16384", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Code Llama (7b) is designed for general code synthesis and understanding." + }, + "codellama/CodeLlama-13b-Instruct-hf": { + "base": "LLAMA", + "type": ["code", "instruct"], + "language": ["en"], + "params": "34b", + "context": "16384", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Code Llama (13b) is designed for general code synthesis and understanding." + }, + "codellama/CodeLlama-34b-Instruct-hf": { + "base": "LLAMA", + "type": ["code", "instruct"], + "language": ["en"], + "params": "34b", + "context": "16384", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Code Llama (34b) is designed for general code synthesis and understanding." + }, + "codellama/CodeLlama-7b-Python-hf": { + "base": "LLAMA", + "type": ["code", "instruct"], + "language": ["en"], + "params": "7b", + "context": "16384", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Code Llama (7b, python) is designed specifically for generating python code." + }, + "codellama/CodeLlama-13b-Python-hf": { + "base": "LLAMA", + "type": ["code", "instruct"], + "language": ["en"], + "params": "13b", + "context": "16384", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Code Llama (13b, python) is designed specifically for generating python code." + }, + "codellama/CodeLlama-34b-Python-hf": { + "base": "LLAMA", + "type": ["code", "instruct"], + "language": ["en"], + "params": "34b", + "context": "16384", + "precision": "bfp16", + "release": "2023", + "benchmark": {}, + "description": "Code Llama (34b, python) is designed specifically for generating python code." + } +} \ No newline at end of file diff --git a/notebook/oai_create_agent.ipynb b/notebook/oai_create_agent.ipynb new file mode 100644 index 00000000000..1309ebad643 --- /dev/null +++ b/notebook/oai_create_agent.ipynb @@ -0,0 +1,124 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Set up an Open-source Agent via vLLM" + ], + "metadata": { + "collapsed": false + }, + "id": "c1004af6a7fbfcd8" + }, + { + "cell_type": "markdown", + "source": [ + "### Requirement\n", + "\n", + "Auto Agent setup need `vllm>=0.2.0`, you can use the following command to install the latest version of [vLLM](https://github.com/vllm-project/vllm).\n", + "```\n", + "pip install vllm\n", + "```" + ], + "metadata": { + "collapsed": false + }, + "id": "ec78dda8e3826d8a" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "!pip install vllm" + ], + "metadata": { + "collapsed": false + }, + "id": "e8e9ae50658be975" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import autogen\n", + "import os" + ], + "metadata": { + "collapsed": false + }, + "id": "8d06a99bd05026bb" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def auto_agent_generation(\n", + " model_name_or_hf_repo: str,\n", + " world_size: int = 4,\n", + " host: str = \"127.0.0.1\",\n", + " port: str = \"8000\",\n", + " logging_path: str = \"logs/\",\n", + " seed: int = 42,\n", + " temperature: int = 0,\n", + "):\n", + " if \"gpt-\" in model_name_or_hf_repo:\n", + " config = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\n", + " \"model\": [model_name_or_hf_repo]\n", + " },\n", + " )\n", + " else:\n", + " model_name = model_name_or_hf_repo.split('/')[-1]\n", + " config = [autogen.get_config_list(\n", + " api_keys = [\"EMPTY\"],\n", + " api_bases = [f\"{host}:{port}\"],\n", + " )]\n", + " # Use vllm to create a server with OpenAI API support\n", + " os.system(f'nohup /root/miniconda3/envs/LLM/bin/python -m vllm.entrypoints.openai.api_server'\n", + " f' --host {host}'\n", + " f' --port {port}'\n", + " f' --model {model_name_or_hf_repo}'\n", + " f' --tensor-parallel-size {world_size} > \"{logging_path}/serving_{model_name}.log\" &')\n", + " \n", + " agent = autogen.AssistantAgent(\n", + " name=\"assistant\",\n", + " llm_config={\n", + " \"seed\": seed, # seed for caching and reproducibility\n", + " \"config_list\": config, # a list of OpenAI API configurations\n", + " \"temperature\": temperature, # temperature for sampling\n", + " }, # configuration for autogen's enhanced inference API which is compatible with OpenAI API\n", + " )\n", + " return agent" + ], + "metadata": { + "collapsed": false + }, + "id": "d729233c9524f875" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 4f79d4887f8a61b1b1ebcf822680b4ba18d486f5 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 28 Oct 2023 18:06:50 +0900 Subject: [PATCH 02/74] complete AgentCreator --- notebook/oai_create_agent.ipynb | 143 +++++++++++++++++++++++--------- 1 file changed, 104 insertions(+), 39 deletions(-) diff --git a/notebook/oai_create_agent.ipynb b/notebook/oai_create_agent.ipynb index 1309ebad643..bb4e962d06f 100644 --- a/notebook/oai_create_agent.ipynb +++ b/notebook/oai_create_agent.ipynb @@ -3,7 +3,7 @@ { "cell_type": "markdown", "source": [ - "# Set up an Open-source Agent via vLLM" + "# Multiple Agent Chat" ], "metadata": { "collapsed": false @@ -43,7 +43,8 @@ "outputs": [], "source": [ "import autogen\n", - "import os" + "import subprocess as sp\n", + "from typing import *" ], "metadata": { "collapsed": false @@ -55,44 +56,108 @@ "execution_count": null, "outputs": [], "source": [ - "def auto_agent_generation(\n", - " model_name_or_hf_repo: str,\n", - " world_size: int = 4,\n", - " host: str = \"127.0.0.1\",\n", - " port: str = \"8000\",\n", - " logging_path: str = \"logs/\",\n", - " seed: int = 42,\n", - " temperature: int = 0,\n", - "):\n", - " if \"gpt-\" in model_name_or_hf_repo:\n", - " config = autogen.config_list_from_json(\n", - " \"OAI_CONFIG_LIST\",\n", - " filter_dict={\n", - " \"model\": [model_name_or_hf_repo]\n", - " },\n", - " )\n", - " else:\n", - " model_name = model_name_or_hf_repo.split('/')[-1]\n", - " config = [autogen.get_config_list(\n", - " api_keys = [\"EMPTY\"],\n", - " api_bases = [f\"{host}:{port}\"],\n", - " )]\n", - " # Use vllm to create a server with OpenAI API support\n", - " os.system(f'nohup /root/miniconda3/envs/LLM/bin/python -m vllm.entrypoints.openai.api_server'\n", - " f' --host {host}'\n", - " f' --port {port}'\n", - " f' --model {model_name_or_hf_repo}'\n", - " f' --tensor-parallel-size {world_size} > \"{logging_path}/serving_{model_name}.log\" &')\n", + "class AgentCreator:\n", + " \n", + " agent_procs: Dict[str, sp.Popen] = {}\n", + " agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {}\n", + " openai_server_name: str = 'openai'\n", + " \n", + " def __init__(\n", + " self,\n", + " logging_path: Optional[str] = \"logs/\",\n", + " ):\n", + " self.logging_path = logging_path\n", + " \n", + " def get_config(\n", + " self,\n", + " model_name_or_hf_repo: str,\n", + " world_size: Optional[int] = 1,\n", + " host: Optional[str] = \"127.0.0.1\",\n", + " port: Optional[str] = \"8000\",\n", + " seed: Optional[int] = 42,\n", + " temperature: Optional[int] = 0,\n", + " ) -> Tuple[List, str]:\n", + " \n", + " if \"gpt-\" in model_name_or_hf_repo:\n", + " config = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\n", + " \"model\": [model_name_or_hf_repo]\n", + " },\n", + " )\n", + " server_id = self.openai_server_name\n", + " else:\n", + " model_name = model_name_or_hf_repo.split('/')[-1]\n", + " config = [autogen.get_config_list(\n", + " api_keys = [\"EMPTY\"],\n", + " api_bases = [f\"{host}:{port}\"],\n", + " )]\n", + " server_id = f'{model_name}_{host}_{port}_{seed}_{temperature}'\n", + " if self.agent_procs.get(server_id, None) is None:\n", + " # Use vllm to set up a server with OpenAI API support\n", + " agent_proc = sp.Popen(['python', '-m', 'vllm.entrypoints.openai.api_server',\n", + " '--host', host,\n", + " '--port', port,\n", + " '--model', model_name_or_hf_repo,\n", + " '--tensor-parallel-size', world_size,\n", + " '>', f\"{self.logging_path}/serving_{model_name}.log\"])\n", + " self.agent_procs[server_id] = agent_proc\n", + " \n", + " return config, server_id\n", " \n", - " agent = autogen.AssistantAgent(\n", - " name=\"assistant\",\n", - " llm_config={\n", - " \"seed\": seed, # seed for caching and reproducibility\n", - " \"config_list\": config, # a list of OpenAI API configurations\n", - " \"temperature\": temperature, # temperature for sampling\n", - " }, # configuration for autogen's enhanced inference API which is compatible with OpenAI API\n", - " )\n", - " return agent" + " def create_agent(\n", + " self, \n", + " agent_name: str,\n", + " seed: int = 42,\n", + " temperature: int = 0,\n", + " **kwargs\n", + " ) -> autogen.AssistantAgent:\n", + " config, server_id = self.get_config(seed=seed, temperature=temperature, **kwargs)\n", + " agent = autogen.AssistantAgent(name=agent_name,\n", + " llm_config={\n", + " \"seed\": seed, # seed for caching and reproducibility\n", + " \"config_list\": config, # a list of OpenAI API configurations\n", + " \"temperature\": temperature, # temperature for sampling\n", + " }, **kwargs)\n", + " self.agent_procs_assign[agent_name] = (agent, server_id)\n", + " return agent\n", + "\n", + " def clear_agent(\n", + " self,\n", + " agent_name: str,\n", + " recycle_endpoint: bool = True\n", + " ):\n", + " _, server_id = self.agent_procs_assign[agent_name]\n", + " del self.agent_procs_assign[agent_name]\n", + " if recycle_endpoint:\n", + " if server_id == self.openai_server_name:\n", + " return\n", + " else:\n", + " for _, iter_sid in self.agent_procs_assign.values():\n", + " if server_id == iter_sid:\n", + " return\n", + " self.agent_procs[server_id].terminate()\n", + " \n", + " def start(self,\n", + " task: str,\n", + " manager_config: dict = {},\n", + " max_round: Optional[int] = 12,\n", + " init_messages: Optional[List[dict]] = [],\n", + " user_name: Optional[str] = \"User Proxy\",\n", + " user_system_message: Optional[str] = \"A human admin.\",\n", + " **kwargs):\n", + " agent_list = [agent for agent, _ in self.agent_procs_assign]\n", + " user_proxy = autogen.UserProxyAgent(\n", + " name=user_name,\n", + " system_message=user_system_message,\n", + " **kwargs\n", + " )\n", + " agent_list.append(user_proxy)\n", + " groupchat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round)\n", + " manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=manager_config)\n", + " \n", + " user_proxy.initiate_chat(manager, message=task)\n", + " " ], "metadata": { "collapsed": false From 73b40612dbd4252ba6029cc28e68e92e6f064969 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sun, 29 Oct 2023 18:49:59 +0900 Subject: [PATCH 03/74] AgentCreator first step features completed. --- notebook/oai_create_agent.ipynb | 659 +++++++++++++++++++++++++++++--- 1 file changed, 596 insertions(+), 63 deletions(-) diff --git a/notebook/oai_create_agent.ipynb b/notebook/oai_create_agent.ipynb index bb4e962d06f..daf51c3fccd 100644 --- a/notebook/oai_create_agent.ipynb +++ b/notebook/oai_create_agent.ipynb @@ -15,9 +15,9 @@ "source": [ "### Requirement\n", "\n", - "Auto Agent setup need `vllm>=0.2.0`, you can use the following command to install the latest version of [vLLM](https://github.com/vllm-project/vllm).\n", + "Auto Agent setup need `vllm>=0.2.0` and `fastchat>=0.2.30`, you can use the following command to install the latest version of [vLLM](https://github.com/vllm-project/vllm) and [FastChat](https://github.com/lm-sys/FastChat).\n", "```\n", - "pip install vllm\n", + "pip install vllm, \"fastchat[model_worker]\"\n", "```" ], "metadata": { @@ -30,7 +30,7 @@ "execution_count": null, "outputs": [], "source": [ - "!pip install vllm" + "!pip install vllm \"fastchat[model_worker]\"" ], "metadata": { "collapsed": false @@ -39,48 +39,76 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "outputs": [], "source": [ "import autogen\n", "import subprocess as sp\n", + "import time\n", "from typing import *" ], "metadata": { - "collapsed": false + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-10-29T09:44:51.998713900Z", + "start_time": "2023-10-29T09:44:51.309000800Z" + } }, "id": "8d06a99bd05026bb" }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "outputs": [], "source": [ "class AgentCreator:\n", - " \n", + " \"\"\"\n", + " Descriptions\n", + " \"\"\"\n", + "\n", " agent_procs: Dict[str, sp.Popen] = {}\n", " agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {}\n", " openai_server_name: str = 'openai'\n", - " \n", + "\n", " def __init__(\n", - " self,\n", - " logging_path: Optional[str] = \"logs/\",\n", + " self,\n", + " endpoint_building_timeout: Optional[int] = 180\n", " ):\n", - " self.logging_path = logging_path\n", - " \n", + " \"\"\"\n", + " Args:\n", + " endpoint_building_timeout: timeout for building up an endpoint server.\n", + " \"\"\"\n", + " self.endpoint_building_timeout = endpoint_building_timeout\n", + "\n", " def get_config(\n", - " self,\n", - " model_name_or_hf_repo: str,\n", - " world_size: Optional[int] = 1,\n", - " host: Optional[str] = \"127.0.0.1\",\n", - " port: Optional[str] = \"8000\",\n", - " seed: Optional[int] = 42,\n", - " temperature: Optional[int] = 0,\n", + " self,\n", + " model_name_or_hf_repo: str,\n", + " world_size: Optional[int] = 1,\n", + " host: Optional[str] = \"127.0.0.1\",\n", + " port: Optional[str] = \"8000\",\n", + " seed: Optional[int] = 42,\n", + " temperature: Optional[int] = 0,\n", + " max_tokens: Optional[int] = 200\n", " ) -> Tuple[List, str]:\n", - " \n", + " \"\"\"\n", + " Descriptions\n", + "\n", + " Args:\n", + " model_name_or_hf_repo: model name (e.g., \"gpt-3.5-turbo\", \"gpt-4\") or huggingface repository\n", + " (e.g., \"meta-llama/Llama-2-7b-chat-hf\") use for starting an endpoint.\n", + " world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs).\n", + " host: the endpoint's API address.\n", + " port: the port of the API address the endpoint listen to.\n", + " seed: random seed for agent.\n", + " temperature: temperature for an agent that use to control the \"preciseness\" of the answer.\n", + " max_tokens: max number of tokens that the Agent can reply.\n", + "\n", + " Returns:\n", + " (agent config list, endpoint server id)\n", + " \"\"\"\n", " if \"gpt-\" in model_name_or_hf_repo:\n", " config = autogen.config_list_from_json(\n", - " \"OAI_CONFIG_LIST\",\n", + " \"/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST\",\n", " filter_dict={\n", " \"model\": [model_name_or_hf_repo]\n", " },\n", @@ -88,45 +116,83 @@ " server_id = self.openai_server_name\n", " else:\n", " model_name = model_name_or_hf_repo.split('/')[-1]\n", - " config = [autogen.get_config_list(\n", - " api_keys = [\"EMPTY\"],\n", - " api_bases = [f\"{host}:{port}\"],\n", - " )]\n", + " config = autogen.get_config_list(\n", + " api_keys=[\"EMPTY\"],\n", + " api_bases=[f\"http://{host}:{port}/v1\"],\n", + " )\n", + " config[0].update({\n", + " \"model\": model_name_or_hf_repo,\n", + " \"max_tokens\": max_tokens\n", + " })\n", " server_id = f'{model_name}_{host}_{port}_{seed}_{temperature}'\n", " if self.agent_procs.get(server_id, None) is None:\n", - " # Use vllm to set up a server with OpenAI API support\n", + " # Use vLLM to set up a server with OpenAI API support.\n", " agent_proc = sp.Popen(['python', '-m', 'vllm.entrypoints.openai.api_server',\n", - " '--host', host,\n", - " '--port', port,\n", - " '--model', model_name_or_hf_repo,\n", - " '--tensor-parallel-size', world_size,\n", - " '>', f\"{self.logging_path}/serving_{model_name}.log\"])\n", + " f'--host', f'{host}',\n", + " f'--port', f'{port}',\n", + " f'--model', f'{model_name_or_hf_repo}',\n", + " f'--tensor-parallel-size', f'{world_size}'], stdout=sp.PIPE, stderr=sp.STDOUT)\n", + " timeout_start = time.time()\n", + " while True:\n", + " server_stdout = agent_proc.stdout.readline()\n", + " timeout_end = time.time()\n", + " if b\"running\" in server_stdout:\n", + " print(f\"Running {model_name_or_hf_repo} endpoint on http://{host}:{port} with tensor parallel size {world_size}.\")\n", + " break\n", + " elif timeout_end - timeout_start > self.endpoint_building_timeout:\n", + " raise RuntimeError(f'Timeout exceed. Fail to set up the endpoint for '\n", + " f'{model_name_or_hf_repo} on {host}:{port}.')\n", " self.agent_procs[server_id] = agent_proc\n", - " \n", + "\n", " return config, server_id\n", - " \n", + "\n", " def create_agent(\n", - " self, \n", - " agent_name: str,\n", - " seed: int = 42,\n", - " temperature: int = 0,\n", - " **kwargs\n", + " self,\n", + " agent_name: str,\n", + " model_name_or_hf_repo: str,\n", + " seed: int = 42,\n", + " temperature: int = 0,\n", + " **kwargs\n", " ) -> autogen.AssistantAgent:\n", - " config, server_id = self.get_config(seed=seed, temperature=temperature, **kwargs)\n", + " \"\"\"\n", + " Descriptions\n", + "\n", + " Args:\n", + " agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...)\n", + " model_name_or_hf_repo: model name (e.g., \"gpt-3.5-turbo\", \"gpt-4\") or huggingface repository\n", + " (e.g., \"meta-llama/Llama-2-7b-chat-hf\") use for starting an endpoint.\n", + " seed: random seed for agent.\n", + " temperature: temperature for an agent that use to control the \"preciseness\" of the answer.\n", + "\n", + " Returns:\n", + " agent: a set-up agent.\n", + " \"\"\"\n", + " config, server_id = self.get_config(model_name_or_hf_repo=model_name_or_hf_repo,\n", + " seed=seed,\n", + " temperature=temperature,\n", + " **kwargs)\n", " agent = autogen.AssistantAgent(name=agent_name,\n", " llm_config={\n", " \"seed\": seed, # seed for caching and reproducibility\n", " \"config_list\": config, # a list of OpenAI API configurations\n", " \"temperature\": temperature, # temperature for sampling\n", - " }, **kwargs)\n", + " })\n", " self.agent_procs_assign[agent_name] = (agent, server_id)\n", " return agent\n", "\n", " def clear_agent(\n", - " self,\n", - " agent_name: str,\n", - " recycle_endpoint: bool = True\n", + " self,\n", + " agent_name: str = None,\n", + " recycle_endpoint: bool = True\n", " ):\n", + " \"\"\"\n", + " Descriptions\n", + "\n", + " Args:\n", + " agent_name: the name of agent.\n", + " recycle_endpoint: trigger for recycle the endpoint server. If true, the endpoint will be recycled\n", + " when there is no agent relying on.\n", + " \"\"\"\n", " _, server_id = self.agent_procs_assign[agent_name]\n", " del self.agent_procs_assign[agent_name]\n", " if recycle_endpoint:\n", @@ -138,31 +204,498 @@ " return\n", " self.agent_procs[server_id].terminate()\n", " \n", - " def start(self,\n", - " task: str,\n", - " manager_config: dict = {},\n", - " max_round: Optional[int] = 12,\n", - " init_messages: Optional[List[dict]] = [],\n", - " user_name: Optional[str] = \"User Proxy\",\n", - " user_system_message: Optional[str] = \"A human admin.\",\n", - " **kwargs):\n", - " agent_list = [agent for agent, _ in self.agent_procs_assign]\n", - " user_proxy = autogen.UserProxyAgent(\n", - " name=user_name,\n", - " system_message=user_system_message,\n", - " **kwargs\n", - " )\n", - " agent_list.append(user_proxy)\n", - " groupchat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round)\n", - " manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=manager_config)\n", - " \n", - " user_proxy.initiate_chat(manager, message=task)\n", + " def clear_all(self):\n", + " \"\"\"\n", + " Clear all cached agents.\n", + " \"\"\"\n", + " for agent_name in [agent_name for agent_name in self.agent_procs_assign.keys()]:\n", + " self.clear_agent(agent_name)\n", + "\n", + " def start(\n", + " self,\n", + " task: str,\n", + " manager_config: dict,\n", + " max_round: Optional[int] = 12,\n", + " init_messages: Optional[List[dict]] = [],\n", + " user_proxy: autogen.UserProxyAgent = None,\n", + " initiate_agent_name: str = \"user\"\n", + " ):\n", + " \"\"\"\n", + " Descriptions\n", + "\n", + " Args:\n", + " task: description of a task.\n", + " manager_config: LLM configuration for a GroupChatManager.\n", + " max_round: the maximum number of rounds.\n", + " init_messages: input messages before the task start. This can be the chat history from other group chat\n", + " or some preliminary of the task.\n", + " user_proxy: a user proxy for human interaction.\n", + " initiate_agent_name: the name of an agent use to initialize the group chat.\n", + " \"\"\"\n", + " agent_list = [agent for agent, _ in self.agent_procs_assign.values()]\n", + " if user_proxy is not None:\n", + " agent_list.append(user_proxy)\n", + " group_chat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round)\n", + " manager = autogen.GroupChatManager(groupchat=group_chat, llm_config=manager_config)\n", + "\n", + " if initiate_agent_name == \"user\" and user_proxy is not None:\n", + " user_proxy.initiate_chat(manager, message=task)\n", + " else:\n", + " for agent in agent_list:\n", + " if initiate_agent_name == agent.name():\n", + " agent.initiate_chat(manager, message=task)\n", " " ], "metadata": { - "collapsed": false + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-10-29T09:48:20.633445Z", + "start_time": "2023-10-29T09:48:20.587240200Z" + } }, "id": "d729233c9524f875" + }, + { + "cell_type": "markdown", + "source": [ + "# Test" + ], + "metadata": { + "collapsed": false + }, + "id": "8f2c3f521de23faa" + }, + { + "cell_type": "code", + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running facebook/opt-125m endpoint on http://127.0.0.1:8000 with tensor parallel size 1.\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", + "\n", + "Here is the Python code to accomplish this task:\n", + "\n", + "```python\n", + "import requests\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response and extract the paper details\n", + "xml_data = response.text\n", + "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", + "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", + "\n", + "# Print the paper details\n", + "print(\"Latest Paper on GPT-4:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Summary:\", paper_summary)\n", + "\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", + "\n", + "```\n", + "\n", + "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "\n", + "When you run the code, it will output the latest paper on GPT-4 from arXiv, including the title and summary. It will also analyze the paper's summary to identify potential applications in software and print them.\n", + "\n", + "Please let me know if you need any further assistance!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager_opt-125m\u001B[0m (to chat_manager):\n", + "\n", + " If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager_opt-125m\u001B[0m (to chat_manager):\n", + "\n", + " If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\n", + "Understood! If you have a specific problem that you would like me to solve using Python code, please provide me with the details of the problem. I'll be happy to help you with a Python solution.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager_opt-125m\u001B[0m (to chat_manager):\n", + "\n", + " If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\n", + "I apologize for the confusion. If you have a specific problem that you would like me to solve using Python code, please provide me with the details of the problem. I'll be happy to help you with a Python solution.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[3], line 15\u001B[0m\n\u001B[1;32m 7\u001B[0m user_proxy \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mUserProxyAgent(\n\u001B[1;32m 8\u001B[0m name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mUser_proxy\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 9\u001B[0m system_message\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mA human admin.\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 10\u001B[0m code_execution_config\u001B[38;5;241m=\u001B[39m{\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mlast_n_messages\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;241m2\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwork_dir\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mgroupchat\u001B[39m\u001B[38;5;124m\"\u001B[39m},\n\u001B[1;32m 11\u001B[0m human_input_mode\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mTERMINATE\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 12\u001B[0m )\n\u001B[1;32m 14\u001B[0m task \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m---> 15\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstart\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 16\u001B[0m \u001B[43m \u001B[49m\u001B[43mtask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 17\u001B[0m \u001B[43m \u001B[49m\u001B[43mmg_config\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 18\u001B[0m \u001B[43m \u001B[49m\u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43muser_proxy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 19\u001B[0m \u001B[43m \u001B[49m\u001B[43minitiate_agent_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43muser\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\n\u001B[1;32m 20\u001B[0m \u001B[43m)\u001B[49m\n", + "Cell \u001B[0;32mIn[2], line 172\u001B[0m, in \u001B[0;36mAgentCreator.start\u001B[0;34m(self, task, manager_config, max_round, init_messages, user_proxy, initiate_agent_name)\u001B[0m\n\u001B[1;32m 169\u001B[0m manager \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mGroupChatManager(groupchat\u001B[38;5;241m=\u001B[39mgroup_chat, llm_config\u001B[38;5;241m=\u001B[39mmanager_config)\n\u001B[1;32m 171\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m initiate_agent_name \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124muser\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mand\u001B[39;00m user_proxy \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 172\u001B[0m \u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minitiate_chat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmanager\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 173\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 174\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m agent \u001B[38;5;129;01min\u001B[39;00m agent_list:\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:531\u001B[0m, in \u001B[0;36mConversableAgent.initiate_chat\u001B[0;34m(self, recipient, clear_history, silent, **context)\u001B[0m\n\u001B[1;32m 517\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"Initiate a chat with the recipient agent.\u001B[39;00m\n\u001B[1;32m 518\u001B[0m \n\u001B[1;32m 519\u001B[0m \u001B[38;5;124;03mReset the consecutive auto reply counter.\u001B[39;00m\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 528\u001B[0m \u001B[38;5;124;03m \"message\" needs to be provided if the `generate_init_message` method is not overridden.\u001B[39;00m\n\u001B[1;32m 529\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m 530\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_chat(recipient, clear_history)\n\u001B[0;32m--> 531\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_init_message\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mcontext\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrecipient\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:334\u001B[0m, in \u001B[0;36mConversableAgent.send\u001B[0;34m(self, message, recipient, request_reply, silent)\u001B[0m\n\u001B[1;32m 332\u001B[0m valid \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_append_oai_message(message, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124massistant\u001B[39m\u001B[38;5;124m\"\u001B[39m, recipient)\n\u001B[1;32m 333\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m valid:\n\u001B[0;32m--> 334\u001B[0m \u001B[43mrecipient\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreceive\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessage\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrequest_reply\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 335\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 336\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 337\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mMessage can\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mt be converted into a valid ChatCompletion message. Either content or function_call must be provided.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 338\u001B[0m )\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:462\u001B[0m, in \u001B[0;36mConversableAgent.receive\u001B[0;34m(self, message, sender, request_reply, silent)\u001B[0m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mreply_at_receive[sender] \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m:\n\u001B[1;32m 461\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m\n\u001B[0;32m--> 462\u001B[0m reply \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_reply\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mchat_messages\u001B[49m\u001B[43m[\u001B[49m\u001B[43msender\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 463\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 464\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msend(reply, sender, silent\u001B[38;5;241m=\u001B[39msilent)\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:781\u001B[0m, in \u001B[0;36mConversableAgent.generate_reply\u001B[0;34m(self, messages, sender, exclude)\u001B[0m\n\u001B[1;32m 779\u001B[0m \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m 780\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_match_trigger(reply_func_tuple[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtrigger\u001B[39m\u001B[38;5;124m\"\u001B[39m], sender):\n\u001B[0;32m--> 781\u001B[0m final, reply \u001B[38;5;241m=\u001B[39m \u001B[43mreply_func\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreply_func_tuple\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mconfig\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 782\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final:\n\u001B[1;32m 783\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m reply\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/groupchat.py:164\u001B[0m, in \u001B[0;36mGroupChatManager.run_chat\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 162\u001B[0m speaker \u001B[38;5;241m=\u001B[39m groupchat\u001B[38;5;241m.\u001B[39mselect_speaker(speaker, \u001B[38;5;28mself\u001B[39m)\n\u001B[1;32m 163\u001B[0m \u001B[38;5;66;03m# let the speaker speak\u001B[39;00m\n\u001B[0;32m--> 164\u001B[0m reply \u001B[38;5;241m=\u001B[39m \u001B[43mspeaker\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_reply\u001B[49m\u001B[43m(\u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 165\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mKeyboardInterrupt\u001B[39;00m:\n\u001B[1;32m 166\u001B[0m \u001B[38;5;66;03m# let the admin agent speak if interrupted\u001B[39;00m\n\u001B[1;32m 167\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m groupchat\u001B[38;5;241m.\u001B[39madmin_name \u001B[38;5;129;01min\u001B[39;00m groupchat\u001B[38;5;241m.\u001B[39magent_names:\n\u001B[1;32m 168\u001B[0m \u001B[38;5;66;03m# admin agent is one of the participants\u001B[39;00m\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:781\u001B[0m, in \u001B[0;36mConversableAgent.generate_reply\u001B[0;34m(self, messages, sender, exclude)\u001B[0m\n\u001B[1;32m 779\u001B[0m \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m 780\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_match_trigger(reply_func_tuple[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtrigger\u001B[39m\u001B[38;5;124m\"\u001B[39m], sender):\n\u001B[0;32m--> 781\u001B[0m final, reply \u001B[38;5;241m=\u001B[39m \u001B[43mreply_func\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreply_func_tuple\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mconfig\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 782\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final:\n\u001B[1;32m 783\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m reply\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:606\u001B[0m, in \u001B[0;36mConversableAgent.generate_oai_reply\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 603\u001B[0m messages \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_oai_messages[sender]\n\u001B[1;32m 605\u001B[0m \u001B[38;5;66;03m# TODO: #1143 handle token limit exceeded error\u001B[39;00m\n\u001B[0;32m--> 606\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43moai\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mChatCompletion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 607\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m-\u001B[39;49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpop\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontext\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_oai_system_message\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mllm_config\u001B[49m\n\u001B[1;32m 608\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 609\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m, oai\u001B[38;5;241m.\u001B[39mChatCompletion\u001B[38;5;241m.\u001B[39mextract_text_or_function_call(response)[\u001B[38;5;241m0\u001B[39m]\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:803\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 801\u001B[0m base_config[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmax_retry_period\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 802\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 803\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 804\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 805\u001B[0m \u001B[43m \u001B[49m\u001B[43muse_cache\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 806\u001B[0m \u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m<\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mlast\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01mor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 807\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mbase_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 808\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 809\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m response \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m 810\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m response\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:834\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 832\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m diskcache\u001B[38;5;241m.\u001B[39mCache(\u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mcache_path) \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39m_cache:\n\u001B[1;32m 833\u001B[0m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mset_cache(seed)\n\u001B[0;32m--> 834\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:224\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 222\u001B[0m response \u001B[38;5;241m=\u001B[39m openai_completion\u001B[38;5;241m.\u001B[39mcreate(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mconfig)\n\u001B[1;32m 223\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 224\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mopenai_completion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 225\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\n\u001B[1;32m 226\u001B[0m ServiceUnavailableError,\n\u001B[1;32m 227\u001B[0m APIConnectionError,\n\u001B[1;32m 228\u001B[0m ):\n\u001B[1;32m 229\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n\u001B[1;32m 230\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mretrying in \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mretry_wait_time\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m seconds...\u001B[39m\u001B[38;5;124m\"\u001B[39m, exc_info\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py:25\u001B[0m, in \u001B[0;36mChatCompletion.create\u001B[0;34m(cls, *args, **kwargs)\u001B[0m\n\u001B[1;32m 23\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 25\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 26\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m TryAgain \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m timeout \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m>\u001B[39m start \u001B[38;5;241m+\u001B[39m timeout:\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001B[0m, in \u001B[0;36mEngineAPIResource.create\u001B[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001B[0m\n\u001B[1;32m 127\u001B[0m \u001B[38;5;129m@classmethod\u001B[39m\n\u001B[1;32m 128\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mcreate\u001B[39m(\n\u001B[1;32m 129\u001B[0m \u001B[38;5;28mcls\u001B[39m,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 136\u001B[0m \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams,\n\u001B[1;32m 137\u001B[0m ):\n\u001B[1;32m 138\u001B[0m (\n\u001B[1;32m 139\u001B[0m deployment_id,\n\u001B[1;32m 140\u001B[0m engine,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 150\u001B[0m api_key, api_base, api_type, api_version, organization, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams\n\u001B[1;32m 151\u001B[0m )\n\u001B[0;32m--> 153\u001B[0m response, _, api_key \u001B[38;5;241m=\u001B[39m \u001B[43mrequestor\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 154\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mpost\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 155\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 156\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 157\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 158\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 159\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 160\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 161\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream:\n\u001B[1;32m 164\u001B[0m \u001B[38;5;66;03m# must be an iterator\u001B[39;00m\n\u001B[1;32m 165\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(response, OpenAIResponse)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:288\u001B[0m, in \u001B[0;36mAPIRequestor.request\u001B[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 277\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mrequest\u001B[39m(\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28mself\u001B[39m,\n\u001B[1;32m 279\u001B[0m method,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 286\u001B[0m request_timeout: Optional[Union[\u001B[38;5;28mfloat\u001B[39m, Tuple[\u001B[38;5;28mfloat\u001B[39m, \u001B[38;5;28mfloat\u001B[39m]]] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m 287\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001B[38;5;28mbool\u001B[39m, \u001B[38;5;28mstr\u001B[39m]:\n\u001B[0;32m--> 288\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest_raw\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 289\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlower\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 290\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 291\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 292\u001B[0m \u001B[43m \u001B[49m\u001B[43msupplied_headers\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 293\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 294\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 295\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 296\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 297\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 298\u001B[0m resp, got_stream \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_interpret_response(result, stream)\n\u001B[1;32m 299\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp, got_stream, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mapi_key\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:596\u001B[0m, in \u001B[0;36mAPIRequestor.request_raw\u001B[0;34m(self, method, url, params, supplied_headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 594\u001B[0m _thread_context\u001B[38;5;241m.\u001B[39msession_create_time \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[1;32m 595\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 596\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 597\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 598\u001B[0m \u001B[43m \u001B[49m\u001B[43mabs_url\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 599\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 600\u001B[0m \u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 601\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 602\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 603\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mif\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01melse\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mTIMEOUT_SECS\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 604\u001B[0m \u001B[43m \u001B[49m\u001B[43mproxies\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mproxies\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 605\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 606\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m requests\u001B[38;5;241m.\u001B[39mexceptions\u001B[38;5;241m.\u001B[39mTimeout \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 607\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m error\u001B[38;5;241m.\u001B[39mTimeout(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mRequest timed out: \u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(e)) \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01me\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:589\u001B[0m, in \u001B[0;36mSession.request\u001B[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001B[0m\n\u001B[1;32m 584\u001B[0m send_kwargs \u001B[38;5;241m=\u001B[39m {\n\u001B[1;32m 585\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimeout\u001B[39m\u001B[38;5;124m\"\u001B[39m: timeout,\n\u001B[1;32m 586\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mallow_redirects\u001B[39m\u001B[38;5;124m\"\u001B[39m: allow_redirects,\n\u001B[1;32m 587\u001B[0m }\n\u001B[1;32m 588\u001B[0m send_kwargs\u001B[38;5;241m.\u001B[39mupdate(settings)\n\u001B[0;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mprep\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43msend_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 591\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:703\u001B[0m, in \u001B[0;36mSession.send\u001B[0;34m(self, request, **kwargs)\u001B[0m\n\u001B[1;32m 700\u001B[0m start \u001B[38;5;241m=\u001B[39m preferred_clock()\n\u001B[1;32m 702\u001B[0m \u001B[38;5;66;03m# Send the request\u001B[39;00m\n\u001B[0;32m--> 703\u001B[0m r \u001B[38;5;241m=\u001B[39m \u001B[43madapter\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 705\u001B[0m \u001B[38;5;66;03m# Total elapsed time of the request (approximately)\u001B[39;00m\n\u001B[1;32m 706\u001B[0m elapsed \u001B[38;5;241m=\u001B[39m preferred_clock() \u001B[38;5;241m-\u001B[39m start\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/adapters.py:486\u001B[0m, in \u001B[0;36mHTTPAdapter.send\u001B[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001B[0m\n\u001B[1;32m 483\u001B[0m timeout \u001B[38;5;241m=\u001B[39m TimeoutSauce(connect\u001B[38;5;241m=\u001B[39mtimeout, read\u001B[38;5;241m=\u001B[39mtimeout)\n\u001B[1;32m 485\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 486\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43murlopen\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 487\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 488\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 489\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 490\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 491\u001B[0m \u001B[43m \u001B[49m\u001B[43mredirect\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 492\u001B[0m \u001B[43m \u001B[49m\u001B[43massert_same_host\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 493\u001B[0m \u001B[43m \u001B[49m\u001B[43mpreload_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 494\u001B[0m \u001B[43m \u001B[49m\u001B[43mdecode_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 495\u001B[0m \u001B[43m \u001B[49m\u001B[43mretries\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmax_retries\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 496\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 497\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 498\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 500\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (ProtocolError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[1;32m 501\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m(err, request\u001B[38;5;241m=\u001B[39mrequest)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:714\u001B[0m, in \u001B[0;36mHTTPConnectionPool.urlopen\u001B[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001B[0m\n\u001B[1;32m 711\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_proxy(conn)\n\u001B[1;32m 713\u001B[0m \u001B[38;5;66;03m# Make the request on the httplib connection object.\u001B[39;00m\n\u001B[0;32m--> 714\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_make_request\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 715\u001B[0m \u001B[43m \u001B[49m\u001B[43mconn\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 716\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 717\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 718\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout_obj\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 719\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 720\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 721\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 722\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 724\u001B[0m \u001B[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001B[39;00m\n\u001B[1;32m 725\u001B[0m \u001B[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001B[39;00m\n\u001B[1;32m 726\u001B[0m \u001B[38;5;66;03m# it will also try to release it and we'll have a double-release\u001B[39;00m\n\u001B[1;32m 727\u001B[0m \u001B[38;5;66;03m# mess.\u001B[39;00m\n\u001B[1;32m 728\u001B[0m response_conn \u001B[38;5;241m=\u001B[39m conn \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m release_conn \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:466\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[0;32m--> 466\u001B[0m \u001B[43msix\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mraise_from\u001B[49m\u001B[43m(\u001B[49m\u001B[43me\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m 467\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 468\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_raise_timeout(err\u001B[38;5;241m=\u001B[39me, url\u001B[38;5;241m=\u001B[39murl, timeout_value\u001B[38;5;241m=\u001B[39mread_timeout)\n", + "File \u001B[0;32m:3\u001B[0m, in \u001B[0;36mraise_from\u001B[0;34m(value, from_value)\u001B[0m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:461\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 458\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n\u001B[1;32m 459\u001B[0m \u001B[38;5;66;03m# Python 3\u001B[39;00m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgetresponse\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[1;32m 466\u001B[0m six\u001B[38;5;241m.\u001B[39mraise_from(e, \u001B[38;5;28;01mNone\u001B[39;00m)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:1375\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 1373\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 1374\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1375\u001B[0m \u001B[43mresponse\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbegin\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1376\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m:\n\u001B[1;32m 1377\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclose()\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:318\u001B[0m, in \u001B[0;36mHTTPResponse.begin\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 316\u001B[0m \u001B[38;5;66;03m# read until we get a non-100 response\u001B[39;00m\n\u001B[1;32m 317\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m--> 318\u001B[0m version, status, reason \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_status\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 319\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m status \u001B[38;5;241m!=\u001B[39m CONTINUE:\n\u001B[1;32m 320\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:279\u001B[0m, in \u001B[0;36mHTTPResponse._read_status\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_read_status\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m--> 279\u001B[0m line \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mstr\u001B[39m(\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreadline\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_MAXLINE\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m)\u001B[49m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124miso-8859-1\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 280\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(line) \u001B[38;5;241m>\u001B[39m _MAXLINE:\n\u001B[1;32m 281\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m LineTooLong(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstatus line\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/socket.py:705\u001B[0m, in \u001B[0;36mSocketIO.readinto\u001B[0;34m(self, b)\u001B[0m\n\u001B[1;32m 703\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 704\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 705\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sock\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrecv_into\u001B[49m\u001B[43m(\u001B[49m\u001B[43mb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 706\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m timeout:\n\u001B[1;32m 707\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_timeout_occurred \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n", + "\u001B[0;31mKeyboardInterrupt\u001B[0m: " + ] + } + ], + "source": [ + "administrator = AgentCreator()\n", + "\n", + "# create agent; get config (manager); initialize user proxy; task start; clear agent\n", + "administrator.create_agent('Coder_gpt-3_5', 'gpt-3.5-turbo', seed=41)\n", + "administrator.create_agent('Product_manager_opt-125m', 'facebook/opt-125m', max_tokens=100, seed=41)\n", + "mg_config, _ = administrator.get_config('gpt-3.5-turbo', seed=41)\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"User_proxy\",\n", + " system_message=\"A human admin.\",\n", + " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", + " human_input_mode=\"TERMINATE\"\n", + ")\n", + "\n", + "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", + "administrator.start(\n", + " task,\n", + " mg_config[0],\n", + " user_proxy=user_proxy,\n", + " initiate_agent_name=\"user\"\n", + ")\n", + "administrator.clear_all()\n" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-10-29T09:37:45.902790700Z", + "start_time": "2023-10-29T09:36:44.499788900Z" + } + }, + "id": "dc7e531b53e5d7d6" + }, + { + "cell_type": "code", + "execution_count": 8, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the abstract or content of the paper to identify its potential applications in software.\n", + "\n", + "Here is the Python code to accomplish this task:\n", + "\n", + "```python\n", + "import requests\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response and extract the paper details\n", + "xml_data = response.text\n", + "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", + "paper_abstract = xml_data.split(\"\")[1].split(\"\")[0]\n", + "\n", + "# Print the paper details\n", + "print(\"Latest Paper on GPT-4:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Abstract:\", paper_abstract)\n", + "```\n", + "\n", + "Please note that the above code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "\n", + "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", + "\n", + "Please let me know if you need any further assistance.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", + "\n", + "I'm sorry for the confusion, but the code I provided is not executable because it's not in a code block. Here is the correct version:\n", + "\n", + "```python\n", + "# filename: arxiv_search.py\n", + "\n", + "import requests\n", + "from bs4 import BeautifulSoup\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response and extract the paper details\n", + "soup = BeautifulSoup(response.content, 'html.parser')\n", + "entry = soup.find('entry')\n", + "\n", + "if entry:\n", + " paper_title = entry.title.text\n", + " paper_abstract = entry.summary.text\n", + "\n", + " # Print the paper details\n", + " print(\"Latest Paper on GPT-4:\")\n", + " print(\"Title:\", paper_title)\n", + " print(\"Abstract:\", paper_abstract)\n", + "else:\n", + " print(\"No papers found related to GPT-4.\")\n", + "```\n", + "\n", + "Please note that the above code uses the `requests` and `BeautifulSoup` libraries to make an HTTP request to the arXiv API and parse the XML response. Make sure you have these libraries installed (`pip install requests beautifulsoup4`) before running the code.\n", + "\n", + "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", + "\n", + "I apologize for the confusion, but the code I provided is not executable because it's not in a code block. Here is the correct version:\n", + "\n", + "```python\n", + "# filename: arxiv_search.py\n", + "\n", + "import requests\n", + "from bs4 import BeautifulSoup\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response and extract the paper details\n", + "soup = BeautifulSoup(response.content, 'html.parser')\n", + "entry = soup.find('entry')\n", + "\n", + "if entry:\n", + " paper_title = entry.title.text\n", + " paper_abstract = entry.summary.text\n", + "\n", + " # Print the paper details\n", + " print(\"Latest Paper on GPT-4:\")\n", + " print(\"Title:\", paper_title)\n", + " print(\"Abstract:\", paper_abstract)\n", + "else:\n", + " print(\"No papers found related to GPT-4.\")\n", + "```\n", + "\n", + "Please note that the above code uses the `requests` and `BeautifulSoup` libraries to make an HTTP request to the arXiv API and parse the XML response. Make sure you have these libraries installed (`pip install requests beautifulsoup4`) before running the code.\n", + "\n", + "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Latest Paper on GPT-4:\n", + "Title: Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models\n", + "Abstract: With LLMs shifting their role from statistical modeling of language to\n", + "serving as general-purpose AI agents, how should LLM evaluations change?\n", + "Arguably, a key ability of an AI agent is to flexibly combine, as needed, the\n", + "basic skills it has learned. The capability to combine skills plays an\n", + "important role in (human) pedagogy and also in a paper on emergence phenomena\n", + "(Arora & Goyal, 2023).\n", + " This work introduces Skill-Mix, a new evaluation to measure ability to\n", + "combine skills. Using a list of $N$ skills the evaluator repeatedly picks\n", + "random subsets of $k$ skills and asks the LLM to produce text combining that\n", + "subset of skills. Since the number of subsets grows like $N^k$, for even modest\n", + "$k$ this evaluation will, with high probability, require the LLM to produce\n", + "text significantly different from any text in the training set. The paper\n", + "develops a methodology for (a) designing and administering such an evaluation,\n", + "and (b) automatic grading (plus spot-checking by humans) of the results using\n", + "GPT-4 as well as the open LLaMA-2 70B model.\n", + " Administering a version of to popular chatbots gave results that, while\n", + "generally in line with prior expectations, contained surprises. Sizeable\n", + "differences exist among model capabilities that are not captured by their\n", + "ranking on popular LLM leaderboards (\"cramming for the leaderboard\").\n", + "Furthermore, simple probability calculations indicate that GPT-4's reasonable\n", + "performance on $k=5$ is suggestive of going beyond \"stochastic parrot\" behavior\n", + "(Bender et al., 2021), i.e., it combines skills in ways that it had not seen\n", + "during training.\n", + " We sketch how the methodology can lead to a Skill-Mix based eco-system of\n", + "open evaluations for AI capabilities of future models.\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", + "\n", + "Based on the abstract of the latest paper titled \"Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models\", the potential applications of GPT-4 in software can be inferred as follows:\n", + "\n", + "1. **General-Purpose AI Agents**: GPT-4 can serve as a general-purpose AI agent, which means it can be used in a wide range of software applications that require AI capabilities. This includes but is not limited to chatbots, virtual assistants, content generation tools, and more.\n", + "\n", + "2. **Combining Skills**: The paper introduces a new evaluation called Skill-Mix to measure the ability to combine skills. This suggests that GPT-4 can be used in software applications where multiple AI skills need to be combined. For example, a software application might require an AI to understand natural language, generate relevant responses, and also exhibit some level of reasoning or decision-making.\n", + "\n", + "3. **Automatic Grading**: The paper discusses automatic grading of the results produced by GPT-4. This suggests potential applications in educational software or any other software that requires automatic grading or evaluation of text.\n", + "\n", + "4. **Beyond \"Stochastic Parrot\" Behavior**: The paper suggests that GPT-4's performance indicates it goes beyond simply repeating patterns it has seen during training. This implies that GPT-4 could be used in software applications that require novel text generation or creative problem-solving.\n", + "\n", + "Please note that these are potential applications inferred from the abstract of the paper. The actual applications could be more specific and varied depending on the detailed content of the paper and the specific requirements of the software.\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "administrator = AgentCreator()\n", + "\n", + "# create agent; get config (manager); initialize user proxy; task start; clear agent\n", + "administrator.create_agent('Coder_gpt-3_5', 'gpt-3.5-turbo')\n", + "administrator.create_agent('Product_manager_gpt-4', 'gpt-4')\n", + "mg_config, _ = administrator.get_config('gpt-3.5-turbo', seed=41)\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"User_proxy\",\n", + " system_message=\"A human admin.\",\n", + " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", + " human_input_mode=\"TERMINATE\"\n", + ")\n", + "\n", + "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", + "administrator.start(\n", + " task,\n", + " mg_config[0],\n", + " user_proxy=user_proxy,\n", + " initiate_agent_name=\"user\"\n", + ")\n", + "administrator.clear_all()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-10-29T09:48:33.383593800Z", + "start_time": "2023-10-29T09:48:32.580332400Z" + } + }, + "id": "7d52e3d9a1bf91cb" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false + }, + "id": "4ca05bc74723e163" } ], "metadata": { From c7e297d80d58cdd0db3d23ddc9d8eaa45adc55c3 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Mon, 30 Oct 2023 04:46:24 +0900 Subject: [PATCH 04/74] update AgentCreator --- notebook/oai_create_agent.ipynb | 400 +++++++++++++++++++------------- 1 file changed, 237 insertions(+), 163 deletions(-) diff --git a/notebook/oai_create_agent.ipynb b/notebook/oai_create_agent.ipynb index daf51c3fccd..0ca441d9451 100644 --- a/notebook/oai_create_agent.ipynb +++ b/notebook/oai_create_agent.ipynb @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "outputs": [], "source": [ "import autogen\n", @@ -50,15 +50,15 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-29T09:44:51.998713900Z", - "start_time": "2023-10-29T09:44:51.309000800Z" + "end_time": "2023-10-29T19:40:21.535921400Z", + "start_time": "2023-10-29T19:40:20.942693900Z" } }, "id": "8d06a99bd05026bb" }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "outputs": [], "source": [ "class AgentCreator:\n", @@ -80,16 +80,14 @@ " \"\"\"\n", " self.endpoint_building_timeout = endpoint_building_timeout\n", "\n", - " def get_config(\n", + " def build_endpoint(\n", " self,\n", " model_name_or_hf_repo: str,\n", " world_size: Optional[int] = 1,\n", " host: Optional[str] = \"127.0.0.1\",\n", " port: Optional[str] = \"8000\",\n", - " seed: Optional[int] = 42,\n", - " temperature: Optional[int] = 0,\n", - " max_tokens: Optional[int] = 200\n", - " ) -> Tuple[List, str]:\n", + " config_path: Optional[str] = './OAI_CONFIG_LIST'\n", + " ) -> Tuple[dict, str]:\n", " \"\"\"\n", " Descriptions\n", "\n", @@ -99,32 +97,23 @@ " world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs).\n", " host: the endpoint's API address.\n", " port: the port of the API address the endpoint listen to.\n", - " seed: random seed for agent.\n", - " temperature: temperature for an agent that use to control the \"preciseness\" of the answer.\n", - " max_tokens: max number of tokens that the Agent can reply.\n", + " config_path: config list path of OpenAI api.\n", "\n", " Returns:\n", " (agent config list, endpoint server id)\n", " \"\"\"\n", + " config_list = autogen.config_list_from_json(\n", + " config_path,\n", + " filter_dict={\n", + " 'model': [model_name_or_hf_repo]\n", + " }\n", + " )\n", " if \"gpt-\" in model_name_or_hf_repo:\n", - " config = autogen.config_list_from_json(\n", - " \"/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST\",\n", - " filter_dict={\n", - " \"model\": [model_name_or_hf_repo]\n", - " },\n", - " )\n", " server_id = self.openai_server_name\n", " else:\n", " model_name = model_name_or_hf_repo.split('/')[-1]\n", - " config = autogen.get_config_list(\n", - " api_keys=[\"EMPTY\"],\n", - " api_bases=[f\"http://{host}:{port}/v1\"],\n", - " )\n", - " config[0].update({\n", - " \"model\": model_name_or_hf_repo,\n", - " \"max_tokens\": max_tokens\n", - " })\n", - " server_id = f'{model_name}_{host}_{port}_{seed}_{temperature}'\n", + " config_list[0]['api_base'] = f\"http://{host}:{port}/v1\"\n", + " server_id = f'{model_name}_{host}_{port}'\n", " if self.agent_procs.get(server_id, None) is None:\n", " # Use vLLM to set up a server with OpenAI API support.\n", " agent_proc = sp.Popen(['python', '-m', 'vllm.entrypoints.openai.api_server',\n", @@ -137,21 +126,20 @@ " server_stdout = agent_proc.stdout.readline()\n", " timeout_end = time.time()\n", " if b\"running\" in server_stdout:\n", - " print(f\"Running {model_name_or_hf_repo} endpoint on http://{host}:{port} with tensor parallel size {world_size}.\")\n", + " print(f\"Running {model_name_or_hf_repo} on http://{host}:{port} with tensor parallel size {world_size}.\")\n", " break\n", " elif timeout_end - timeout_start > self.endpoint_building_timeout:\n", " raise RuntimeError(f'Timeout exceed. Fail to set up the endpoint for '\n", " f'{model_name_or_hf_repo} on {host}:{port}.')\n", " self.agent_procs[server_id] = agent_proc\n", "\n", - " return config, server_id\n", + " return config_list, server_id\n", "\n", " def create_agent(\n", " self,\n", " agent_name: str,\n", - " model_name_or_hf_repo: str,\n", - " seed: int = 42,\n", - " temperature: int = 0,\n", + " llm_config: dict,\n", + " server_id: str,\n", " **kwargs\n", " ) -> autogen.AssistantAgent:\n", " \"\"\"\n", @@ -159,24 +147,15 @@ "\n", " Args:\n", " agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...)\n", - " model_name_or_hf_repo: model name (e.g., \"gpt-3.5-turbo\", \"gpt-4\") or huggingface repository\n", - " (e.g., \"meta-llama/Llama-2-7b-chat-hf\") use for starting an endpoint.\n", - " seed: random seed for agent.\n", - " temperature: temperature for an agent that use to control the \"preciseness\" of the answer.\n", + " llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...).\n", + " server_id: endpoint server id.\n", "\n", " Returns:\n", " agent: a set-up agent.\n", " \"\"\"\n", - " config, server_id = self.get_config(model_name_or_hf_repo=model_name_or_hf_repo,\n", - " seed=seed,\n", - " temperature=temperature,\n", - " **kwargs)\n", " agent = autogen.AssistantAgent(name=agent_name,\n", - " llm_config={\n", - " \"seed\": seed, # seed for caching and reproducibility\n", - " \"config_list\": config, # a list of OpenAI API configurations\n", - " \"temperature\": temperature, # temperature for sampling\n", - " })\n", + " llm_config=llm_config,\n", + " **kwargs)\n", " self.agent_procs_assign[agent_name] = (agent, server_id)\n", " return agent\n", "\n", @@ -191,7 +170,7 @@ " Args:\n", " agent_name: the name of agent.\n", " recycle_endpoint: trigger for recycle the endpoint server. If true, the endpoint will be recycled\n", - " when there is no agent relying on.\n", + " when there is no agent depending on.\n", " \"\"\"\n", " _, server_id = self.agent_procs_assign[agent_name]\n", " del self.agent_procs_assign[agent_name]\n", @@ -214,7 +193,8 @@ " def start(\n", " self,\n", " task: str,\n", - " manager_config: dict,\n", + " manager_llm_config: dict,\n", + " manager_server_id: str,\n", " max_round: Optional[int] = 12,\n", " init_messages: Optional[List[dict]] = [],\n", " user_proxy: autogen.UserProxyAgent = None,\n", @@ -224,8 +204,9 @@ " Descriptions\n", "\n", " Args:\n", - " task: description of a task.\n", - " manager_config: LLM configuration for a GroupChatManager.\n", + " task: task description.\n", + " manager_llm_config: LLM configuration for a GroupChatManager.\n", + " manager_server_id: id of the server the manager depending on.\n", " max_round: the maximum number of rounds.\n", " init_messages: input messages before the task start. This can be the chat history from other group chat\n", " or some preliminary of the task.\n", @@ -236,7 +217,8 @@ " if user_proxy is not None:\n", " agent_list.append(user_proxy)\n", " group_chat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round)\n", - " manager = autogen.GroupChatManager(groupchat=group_chat, llm_config=manager_config)\n", + " manager = autogen.GroupChatManager(groupchat=group_chat, llm_config=manager_llm_config)\n", + " self.agent_procs_assign['manager'] = (manager, manager_server_id)\n", "\n", " if initiate_agent_name == \"user\" and user_proxy is not None:\n", " user_proxy.initiate_chat(manager, message=task)\n", @@ -244,13 +226,14 @@ " for agent in agent_list:\n", " if initiate_agent_name == agent.name():\n", " agent.initiate_chat(manager, message=task)\n", - " " + " \n", + " self.clear_agent('manager')" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-29T09:48:20.633445Z", - "start_time": "2023-10-29T09:48:20.587240200Z" + "end_time": "2023-10-29T19:40:21.588597Z", + "start_time": "2023-10-29T19:40:21.544449200Z" } }, "id": "d729233c9524f875" @@ -273,8 +256,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running facebook/opt-125m endpoint on http://127.0.0.1:8000 with tensor parallel size 1.\n", + "Running facebook/opt-125m on http://127.0.0.1:8000 with tensor parallel size 1.\n", "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", @@ -324,19 +308,15 @@ "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_opt-125m\u001B[0m (to chat_manager):\n", - "\n", - " If you need to solve a problem with a python code, please use the python code to solve the problem.\n", "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", "\n", - "Understood! If you have a specific problem that you would like me to solve using Python code, please provide me with the details of the problem. I'll be happy to help you with a Python solution.\n", + "I apologize for the repeated message. If you have a specific problem that you would like to solve using Python code, please provide the details of the problem and I will be happy to assist you with a Python solution.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mProduct_manager_opt-125m\u001B[0m (to chat_manager):\n", @@ -344,14 +324,51 @@ " If you need to solve a problem with a python code, please use the python code to solve the problem.\n", "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", + "### Human: If you need to solve a problem with a python code, please use the python code\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", "\n", - "I apologize for the confusion. If you have a specific problem that you would like me to solve using Python code, please provide me with the details of the problem. I'll be happy to help you with a Python solution.\n", "\n", - "--------------------------------------------------------------------------------\n" + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "[autogen.oai.completion: 10-30 04:38:04] {238} INFO - retrying in 10 seconds...\n", + "Traceback (most recent call last):\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 403, in handle_error_response\n", + " error_data = resp[\"error\"]\n", + "KeyError: 'error'\n", + "\n", + "During handling of the above exception, another exception occurred:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/home/elpis_ubuntu/LLM/autogen/autogen/oai/completion.py\", line 224, in _get_response\n", + " response = openai_completion.create(request_timeout=request_timeout, **config)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py\", line 25, in create\n", + " return super().create(*args, **kwargs)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py\", line 153, in create\n", + " response, _, api_key = requestor.request(\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 298, in request\n", + " resp, got_stream = self._interpret_response(result, stream)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 700, in _interpret_response\n", + " self._interpret_response_line(\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 763, in _interpret_response_line\n", + " raise self.handle_error_response(\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 405, in handle_error_response\n", + " raise error.APIError(\n", + "openai.error.APIError: Invalid response object from API: '{\"object\":\"error\",\"message\":\"This model\\'s maximum context length is 2048 tokens. However, you requested 2092 tokens (1892 in the messages, 200 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":null}' (HTTP response code was 400)\n" ] }, { @@ -360,9 +377,23 @@ "output_type": "error", "traceback": [ "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mKeyError\u001B[0m Traceback (most recent call last)", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:403\u001B[0m, in \u001B[0;36mAPIRequestor.handle_error_response\u001B[0;34m(self, rbody, rcode, resp, rheaders, stream_error)\u001B[0m\n\u001B[1;32m 402\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 403\u001B[0m error_data \u001B[38;5;241m=\u001B[39m \u001B[43mresp\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43merror\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\n\u001B[1;32m 404\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\u001B[38;5;167;01mKeyError\u001B[39;00m, \u001B[38;5;167;01mTypeError\u001B[39;00m):\n", + "\u001B[0;31mKeyError\u001B[0m: 'error'", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001B[0;31mAPIError\u001B[0m Traceback (most recent call last)", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:224\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 223\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 224\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mopenai_completion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 225\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\n\u001B[1;32m 226\u001B[0m ServiceUnavailableError,\n\u001B[1;32m 227\u001B[0m APIConnectionError,\n\u001B[1;32m 228\u001B[0m ):\n\u001B[1;32m 229\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py:25\u001B[0m, in \u001B[0;36mChatCompletion.create\u001B[0;34m(cls, *args, **kwargs)\u001B[0m\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 25\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 26\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m TryAgain \u001B[38;5;28;01mas\u001B[39;00m e:\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001B[0m, in \u001B[0;36mEngineAPIResource.create\u001B[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001B[0m\n\u001B[1;32m 138\u001B[0m (\n\u001B[1;32m 139\u001B[0m deployment_id,\n\u001B[1;32m 140\u001B[0m engine,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 150\u001B[0m api_key, api_base, api_type, api_version, organization, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams\n\u001B[1;32m 151\u001B[0m )\n\u001B[0;32m--> 153\u001B[0m response, _, api_key \u001B[38;5;241m=\u001B[39m \u001B[43mrequestor\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 154\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mpost\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 155\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 156\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 157\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 158\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 159\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 160\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 161\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream:\n\u001B[1;32m 164\u001B[0m \u001B[38;5;66;03m# must be an iterator\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:298\u001B[0m, in \u001B[0;36mAPIRequestor.request\u001B[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 288\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mrequest_raw(\n\u001B[1;32m 289\u001B[0m method\u001B[38;5;241m.\u001B[39mlower(),\n\u001B[1;32m 290\u001B[0m url,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 296\u001B[0m request_timeout\u001B[38;5;241m=\u001B[39mrequest_timeout,\n\u001B[1;32m 297\u001B[0m )\n\u001B[0;32m--> 298\u001B[0m resp, got_stream \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_interpret_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mresult\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 299\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp, got_stream, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mapi_key\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:700\u001B[0m, in \u001B[0;36mAPIRequestor._interpret_response\u001B[0;34m(self, result, stream)\u001B[0m\n\u001B[1;32m 698\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 699\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m (\n\u001B[0;32m--> 700\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_interpret_response_line\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 701\u001B[0m \u001B[43m \u001B[49m\u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcontent\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdecode\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mutf-8\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 702\u001B[0m \u001B[43m \u001B[49m\u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstatus_code\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 703\u001B[0m \u001B[43m \u001B[49m\u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 704\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 705\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m,\n\u001B[1;32m 706\u001B[0m \u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[1;32m 707\u001B[0m )\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:763\u001B[0m, in \u001B[0;36mAPIRequestor._interpret_response_line\u001B[0;34m(self, rbody, rcode, rheaders, stream)\u001B[0m\n\u001B[1;32m 762\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream_error \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;241m200\u001B[39m \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m rcode \u001B[38;5;241m<\u001B[39m \u001B[38;5;241m300\u001B[39m:\n\u001B[0;32m--> 763\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhandle_error_response\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 764\u001B[0m \u001B[43m \u001B[49m\u001B[43mrbody\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrcode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mresp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrheaders\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstream_error\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream_error\u001B[49m\n\u001B[1;32m 765\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 766\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:405\u001B[0m, in \u001B[0;36mAPIRequestor.handle_error_response\u001B[0;34m(self, rbody, rcode, resp, rheaders, stream_error)\u001B[0m\n\u001B[1;32m 404\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\u001B[38;5;167;01mKeyError\u001B[39;00m, \u001B[38;5;167;01mTypeError\u001B[39;00m):\n\u001B[0;32m--> 405\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m error\u001B[38;5;241m.\u001B[39mAPIError(\n\u001B[1;32m 406\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mInvalid response object from API: \u001B[39m\u001B[38;5;132;01m%r\u001B[39;00m\u001B[38;5;124m (HTTP response code \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 407\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwas \u001B[39m\u001B[38;5;132;01m%d\u001B[39;00m\u001B[38;5;124m)\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m (rbody, rcode),\n\u001B[1;32m 408\u001B[0m rbody,\n\u001B[1;32m 409\u001B[0m rcode,\n\u001B[1;32m 410\u001B[0m resp,\n\u001B[1;32m 411\u001B[0m )\n\u001B[1;32m 413\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124minternal_message\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m error_data:\n", + "\u001B[0;31mAPIError\u001B[0m: Invalid response object from API: '{\"object\":\"error\",\"message\":\"This model\\'s maximum context length is 2048 tokens. However, you requested 2092 tokens (1892 in the messages, 200 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":null}' (HTTP response code was 400)", + "\nDuring handling of the above exception, another exception occurred:\n", "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[3], line 15\u001B[0m\n\u001B[1;32m 7\u001B[0m user_proxy \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mUserProxyAgent(\n\u001B[1;32m 8\u001B[0m name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mUser_proxy\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 9\u001B[0m system_message\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mA human admin.\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 10\u001B[0m code_execution_config\u001B[38;5;241m=\u001B[39m{\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mlast_n_messages\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;241m2\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwork_dir\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mgroupchat\u001B[39m\u001B[38;5;124m\"\u001B[39m},\n\u001B[1;32m 11\u001B[0m human_input_mode\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mTERMINATE\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 12\u001B[0m )\n\u001B[1;32m 14\u001B[0m task \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m---> 15\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstart\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 16\u001B[0m \u001B[43m \u001B[49m\u001B[43mtask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 17\u001B[0m \u001B[43m \u001B[49m\u001B[43mmg_config\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 18\u001B[0m \u001B[43m \u001B[49m\u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43muser_proxy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 19\u001B[0m \u001B[43m \u001B[49m\u001B[43minitiate_agent_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43muser\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\n\u001B[1;32m 20\u001B[0m \u001B[43m)\u001B[49m\n", - "Cell \u001B[0;32mIn[2], line 172\u001B[0m, in \u001B[0;36mAgentCreator.start\u001B[0;34m(self, task, manager_config, max_round, init_messages, user_proxy, initiate_agent_name)\u001B[0m\n\u001B[1;32m 169\u001B[0m manager \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mGroupChatManager(groupchat\u001B[38;5;241m=\u001B[39mgroup_chat, llm_config\u001B[38;5;241m=\u001B[39mmanager_config)\n\u001B[1;32m 171\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m initiate_agent_name \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124muser\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mand\u001B[39;00m user_proxy \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 172\u001B[0m \u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minitiate_chat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmanager\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 173\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 174\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m agent \u001B[38;5;129;01min\u001B[39;00m agent_list:\n", + "Cell \u001B[0;32mIn[3], line 32\u001B[0m\n\u001B[1;32m 24\u001B[0m user_proxy \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mUserProxyAgent(\n\u001B[1;32m 25\u001B[0m name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mUser_proxy\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 26\u001B[0m system_message\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mA human admin.\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 27\u001B[0m code_execution_config\u001B[38;5;241m=\u001B[39m{\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mlast_n_messages\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;241m2\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwork_dir\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mgroupchat\u001B[39m\u001B[38;5;124m\"\u001B[39m},\n\u001B[1;32m 28\u001B[0m human_input_mode\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mTERMINATE\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 29\u001B[0m )\n\u001B[1;32m 31\u001B[0m task \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m---> 32\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstart\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 33\u001B[0m \u001B[43m \u001B[49m\u001B[43mtask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 34\u001B[0m \u001B[43m \u001B[49m\u001B[43mmg_llm_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 35\u001B[0m \u001B[43m \u001B[49m\u001B[43mgpt3_5_server_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 36\u001B[0m \u001B[43m \u001B[49m\u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43muser_proxy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 37\u001B[0m \u001B[43m \u001B[49m\u001B[43minitiate_agent_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43muser\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\n\u001B[1;32m 38\u001B[0m \u001B[43m)\u001B[49m\n\u001B[1;32m 39\u001B[0m administrator\u001B[38;5;241m.\u001B[39mclear_all()\n", + "Cell \u001B[0;32mIn[2], line 161\u001B[0m, in \u001B[0;36mAgentCreator.start\u001B[0;34m(self, task, manager_llm_config, manager_server_id, max_round, init_messages, user_proxy, initiate_agent_name)\u001B[0m\n\u001B[1;32m 158\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39magent_procs_assign[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmanager\u001B[39m\u001B[38;5;124m'\u001B[39m] \u001B[38;5;241m=\u001B[39m (manager, manager_server_id)\n\u001B[1;32m 160\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m initiate_agent_name \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124muser\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mand\u001B[39;00m user_proxy \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 161\u001B[0m \u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minitiate_chat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmanager\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 162\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m agent \u001B[38;5;129;01min\u001B[39;00m agent_list:\n", "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:531\u001B[0m, in \u001B[0;36mConversableAgent.initiate_chat\u001B[0;34m(self, recipient, clear_history, silent, **context)\u001B[0m\n\u001B[1;32m 517\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"Initiate a chat with the recipient agent.\u001B[39;00m\n\u001B[1;32m 518\u001B[0m \n\u001B[1;32m 519\u001B[0m \u001B[38;5;124;03mReset the consecutive auto reply counter.\u001B[39;00m\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 528\u001B[0m \u001B[38;5;124;03m \"message\" needs to be provided if the `generate_init_message` method is not overridden.\u001B[39;00m\n\u001B[1;32m 529\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m 530\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_chat(recipient, clear_history)\n\u001B[0;32m--> 531\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_init_message\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mcontext\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrecipient\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n", "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:334\u001B[0m, in \u001B[0;36mConversableAgent.send\u001B[0;34m(self, message, recipient, request_reply, silent)\u001B[0m\n\u001B[1;32m 332\u001B[0m valid \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_append_oai_message(message, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124massistant\u001B[39m\u001B[38;5;124m\"\u001B[39m, recipient)\n\u001B[1;32m 333\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m valid:\n\u001B[0;32m--> 334\u001B[0m \u001B[43mrecipient\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreceive\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessage\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrequest_reply\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 335\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 336\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 337\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mMessage can\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mt be converted into a valid ChatCompletion message. Either content or function_call must be provided.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 338\u001B[0m )\n", "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:462\u001B[0m, in \u001B[0;36mConversableAgent.receive\u001B[0;34m(self, message, sender, request_reply, silent)\u001B[0m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mreply_at_receive[sender] \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m:\n\u001B[1;32m 461\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m\n\u001B[0;32m--> 462\u001B[0m reply \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_reply\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mchat_messages\u001B[49m\u001B[43m[\u001B[49m\u001B[43msender\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 463\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 464\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msend(reply, sender, silent\u001B[38;5;241m=\u001B[39msilent)\n", @@ -372,33 +403,36 @@ "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:606\u001B[0m, in \u001B[0;36mConversableAgent.generate_oai_reply\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 603\u001B[0m messages \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_oai_messages[sender]\n\u001B[1;32m 605\u001B[0m \u001B[38;5;66;03m# TODO: #1143 handle token limit exceeded error\u001B[39;00m\n\u001B[0;32m--> 606\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43moai\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mChatCompletion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 607\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m-\u001B[39;49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpop\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontext\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_oai_system_message\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mllm_config\u001B[49m\n\u001B[1;32m 608\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 609\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m, oai\u001B[38;5;241m.\u001B[39mChatCompletion\u001B[38;5;241m.\u001B[39mextract_text_or_function_call(response)[\u001B[38;5;241m0\u001B[39m]\n", "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:803\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 801\u001B[0m base_config[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmax_retry_period\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 802\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 803\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 804\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 805\u001B[0m \u001B[43m \u001B[49m\u001B[43muse_cache\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 806\u001B[0m \u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m<\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mlast\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01mor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 807\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mbase_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 808\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 809\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m response \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m 810\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m response\n", "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:834\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 832\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m diskcache\u001B[38;5;241m.\u001B[39mCache(\u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mcache_path) \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39m_cache:\n\u001B[1;32m 833\u001B[0m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mset_cache(seed)\n\u001B[0;32m--> 834\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:224\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 222\u001B[0m response \u001B[38;5;241m=\u001B[39m openai_completion\u001B[38;5;241m.\u001B[39mcreate(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mconfig)\n\u001B[1;32m 223\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 224\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mopenai_completion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 225\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\n\u001B[1;32m 226\u001B[0m ServiceUnavailableError,\n\u001B[1;32m 227\u001B[0m APIConnectionError,\n\u001B[1;32m 228\u001B[0m ):\n\u001B[1;32m 229\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n\u001B[1;32m 230\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mretrying in \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mretry_wait_time\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m seconds...\u001B[39m\u001B[38;5;124m\"\u001B[39m, exc_info\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py:25\u001B[0m, in \u001B[0;36mChatCompletion.create\u001B[0;34m(cls, *args, **kwargs)\u001B[0m\n\u001B[1;32m 23\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 25\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 26\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m TryAgain \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m timeout \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m>\u001B[39m start \u001B[38;5;241m+\u001B[39m timeout:\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001B[0m, in \u001B[0;36mEngineAPIResource.create\u001B[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001B[0m\n\u001B[1;32m 127\u001B[0m \u001B[38;5;129m@classmethod\u001B[39m\n\u001B[1;32m 128\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mcreate\u001B[39m(\n\u001B[1;32m 129\u001B[0m \u001B[38;5;28mcls\u001B[39m,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 136\u001B[0m \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams,\n\u001B[1;32m 137\u001B[0m ):\n\u001B[1;32m 138\u001B[0m (\n\u001B[1;32m 139\u001B[0m deployment_id,\n\u001B[1;32m 140\u001B[0m engine,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 150\u001B[0m api_key, api_base, api_type, api_version, organization, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams\n\u001B[1;32m 151\u001B[0m )\n\u001B[0;32m--> 153\u001B[0m response, _, api_key \u001B[38;5;241m=\u001B[39m \u001B[43mrequestor\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 154\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mpost\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 155\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 156\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 157\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 158\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 159\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 160\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 161\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream:\n\u001B[1;32m 164\u001B[0m \u001B[38;5;66;03m# must be an iterator\u001B[39;00m\n\u001B[1;32m 165\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(response, OpenAIResponse)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:288\u001B[0m, in \u001B[0;36mAPIRequestor.request\u001B[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 277\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mrequest\u001B[39m(\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28mself\u001B[39m,\n\u001B[1;32m 279\u001B[0m method,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 286\u001B[0m request_timeout: Optional[Union[\u001B[38;5;28mfloat\u001B[39m, Tuple[\u001B[38;5;28mfloat\u001B[39m, \u001B[38;5;28mfloat\u001B[39m]]] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m 287\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001B[38;5;28mbool\u001B[39m, \u001B[38;5;28mstr\u001B[39m]:\n\u001B[0;32m--> 288\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest_raw\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 289\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlower\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 290\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 291\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 292\u001B[0m \u001B[43m \u001B[49m\u001B[43msupplied_headers\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 293\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 294\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 295\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 296\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 297\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 298\u001B[0m resp, got_stream \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_interpret_response(result, stream)\n\u001B[1;32m 299\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp, got_stream, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mapi_key\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:596\u001B[0m, in \u001B[0;36mAPIRequestor.request_raw\u001B[0;34m(self, method, url, params, supplied_headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 594\u001B[0m _thread_context\u001B[38;5;241m.\u001B[39msession_create_time \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[1;32m 595\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 596\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 597\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 598\u001B[0m \u001B[43m \u001B[49m\u001B[43mabs_url\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 599\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 600\u001B[0m \u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 601\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 602\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 603\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mif\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01melse\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mTIMEOUT_SECS\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 604\u001B[0m \u001B[43m \u001B[49m\u001B[43mproxies\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mproxies\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 605\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 606\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m requests\u001B[38;5;241m.\u001B[39mexceptions\u001B[38;5;241m.\u001B[39mTimeout \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 607\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m error\u001B[38;5;241m.\u001B[39mTimeout(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mRequest timed out: \u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(e)) \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01me\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:589\u001B[0m, in \u001B[0;36mSession.request\u001B[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001B[0m\n\u001B[1;32m 584\u001B[0m send_kwargs \u001B[38;5;241m=\u001B[39m {\n\u001B[1;32m 585\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimeout\u001B[39m\u001B[38;5;124m\"\u001B[39m: timeout,\n\u001B[1;32m 586\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mallow_redirects\u001B[39m\u001B[38;5;124m\"\u001B[39m: allow_redirects,\n\u001B[1;32m 587\u001B[0m }\n\u001B[1;32m 588\u001B[0m send_kwargs\u001B[38;5;241m.\u001B[39mupdate(settings)\n\u001B[0;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mprep\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43msend_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 591\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:703\u001B[0m, in \u001B[0;36mSession.send\u001B[0;34m(self, request, **kwargs)\u001B[0m\n\u001B[1;32m 700\u001B[0m start \u001B[38;5;241m=\u001B[39m preferred_clock()\n\u001B[1;32m 702\u001B[0m \u001B[38;5;66;03m# Send the request\u001B[39;00m\n\u001B[0;32m--> 703\u001B[0m r \u001B[38;5;241m=\u001B[39m \u001B[43madapter\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 705\u001B[0m \u001B[38;5;66;03m# Total elapsed time of the request (approximately)\u001B[39;00m\n\u001B[1;32m 706\u001B[0m elapsed \u001B[38;5;241m=\u001B[39m preferred_clock() \u001B[38;5;241m-\u001B[39m start\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/adapters.py:486\u001B[0m, in \u001B[0;36mHTTPAdapter.send\u001B[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001B[0m\n\u001B[1;32m 483\u001B[0m timeout \u001B[38;5;241m=\u001B[39m TimeoutSauce(connect\u001B[38;5;241m=\u001B[39mtimeout, read\u001B[38;5;241m=\u001B[39mtimeout)\n\u001B[1;32m 485\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 486\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43murlopen\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 487\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 488\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 489\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 490\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 491\u001B[0m \u001B[43m \u001B[49m\u001B[43mredirect\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 492\u001B[0m \u001B[43m \u001B[49m\u001B[43massert_same_host\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 493\u001B[0m \u001B[43m \u001B[49m\u001B[43mpreload_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 494\u001B[0m \u001B[43m \u001B[49m\u001B[43mdecode_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 495\u001B[0m \u001B[43m \u001B[49m\u001B[43mretries\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmax_retries\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 496\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 497\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 498\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 500\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (ProtocolError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[1;32m 501\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m(err, request\u001B[38;5;241m=\u001B[39mrequest)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:714\u001B[0m, in \u001B[0;36mHTTPConnectionPool.urlopen\u001B[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001B[0m\n\u001B[1;32m 711\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_proxy(conn)\n\u001B[1;32m 713\u001B[0m \u001B[38;5;66;03m# Make the request on the httplib connection object.\u001B[39;00m\n\u001B[0;32m--> 714\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_make_request\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 715\u001B[0m \u001B[43m \u001B[49m\u001B[43mconn\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 716\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 717\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 718\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout_obj\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 719\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 720\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 721\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 722\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 724\u001B[0m \u001B[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001B[39;00m\n\u001B[1;32m 725\u001B[0m \u001B[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001B[39;00m\n\u001B[1;32m 726\u001B[0m \u001B[38;5;66;03m# it will also try to release it and we'll have a double-release\u001B[39;00m\n\u001B[1;32m 727\u001B[0m \u001B[38;5;66;03m# mess.\u001B[39;00m\n\u001B[1;32m 728\u001B[0m response_conn \u001B[38;5;241m=\u001B[39m conn \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m release_conn \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:466\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[0;32m--> 466\u001B[0m \u001B[43msix\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mraise_from\u001B[49m\u001B[43m(\u001B[49m\u001B[43me\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m 467\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 468\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_raise_timeout(err\u001B[38;5;241m=\u001B[39me, url\u001B[38;5;241m=\u001B[39murl, timeout_value\u001B[38;5;241m=\u001B[39mread_timeout)\n", - "File \u001B[0;32m:3\u001B[0m, in \u001B[0;36mraise_from\u001B[0;34m(value, from_value)\u001B[0m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:461\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 458\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n\u001B[1;32m 459\u001B[0m \u001B[38;5;66;03m# Python 3\u001B[39;00m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgetresponse\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[1;32m 466\u001B[0m six\u001B[38;5;241m.\u001B[39mraise_from(e, \u001B[38;5;28;01mNone\u001B[39;00m)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:1375\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 1373\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 1374\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1375\u001B[0m \u001B[43mresponse\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbegin\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1376\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m:\n\u001B[1;32m 1377\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclose()\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:318\u001B[0m, in \u001B[0;36mHTTPResponse.begin\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 316\u001B[0m \u001B[38;5;66;03m# read until we get a non-100 response\u001B[39;00m\n\u001B[1;32m 317\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m--> 318\u001B[0m version, status, reason \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_status\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 319\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m status \u001B[38;5;241m!=\u001B[39m CONTINUE:\n\u001B[1;32m 320\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:279\u001B[0m, in \u001B[0;36mHTTPResponse._read_status\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_read_status\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m--> 279\u001B[0m line \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mstr\u001B[39m(\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreadline\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_MAXLINE\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m)\u001B[49m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124miso-8859-1\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 280\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(line) \u001B[38;5;241m>\u001B[39m _MAXLINE:\n\u001B[1;32m 281\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m LineTooLong(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstatus line\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/socket.py:705\u001B[0m, in \u001B[0;36mSocketIO.readinto\u001B[0;34m(self, b)\u001B[0m\n\u001B[1;32m 703\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 704\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 705\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sock\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrecv_into\u001B[49m\u001B[43m(\u001B[49m\u001B[43mb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 706\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m timeout:\n\u001B[1;32m 707\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_timeout_occurred \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:239\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 237\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n\u001B[1;32m 238\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mretrying in \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mretry_wait_time\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m seconds...\u001B[39m\u001B[38;5;124m\"\u001B[39m, exc_info\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m)\n\u001B[0;32m--> 239\u001B[0m \u001B[43msleep\u001B[49m\u001B[43m(\u001B[49m\u001B[43mretry_wait_time\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 240\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (RateLimitError, Timeout) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[1;32m 241\u001B[0m time_left \u001B[38;5;241m=\u001B[39m max_retry_period \u001B[38;5;241m-\u001B[39m (time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m-\u001B[39m start_time \u001B[38;5;241m+\u001B[39m retry_wait_time)\n", "\u001B[0;31mKeyboardInterrupt\u001B[0m: " ] } ], "source": [ "administrator = AgentCreator()\n", + "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", + "\n", + "# build endpoint; create agent; get config (manager); initialize user proxy; task start; clear agent\n", + "gpt3_5_config_list, gpt3_5_server_id = administrator.build_endpoint('gpt-3.5-turbo', config_path=config_path)\n", + "opt_config_list, opt_server_id = administrator.build_endpoint('facebook/opt-125m', config_path=config_path)\n", + "\n", + "coder_llm_config = {\n", + " 'config_list': gpt3_5_config_list,\n", + " 'temperature': 0,\n", + "}\n", + "pm_llm_config = {\n", + " 'config_list': opt_config_list,\n", + " 'temperature': 0,\n", + " 'max_tokens': 200\n", + "}\n", + "mg_llm_config = {\n", + " 'config_list': gpt3_5_config_list,\n", + " 'temperature': 0,\n", + "}\n", + "\n", + "administrator.create_agent('Coder_gpt-3_5', coder_llm_config, gpt3_5_server_id)\n", + "administrator.create_agent('Product_manager_opt-125m', pm_llm_config, opt_server_id)\n", "\n", - "# create agent; get config (manager); initialize user proxy; task start; clear agent\n", - "administrator.create_agent('Coder_gpt-3_5', 'gpt-3.5-turbo', seed=41)\n", - "administrator.create_agent('Product_manager_opt-125m', 'facebook/opt-125m', max_tokens=100, seed=41)\n", - "mg_config, _ = administrator.get_config('gpt-3.5-turbo', seed=41)\n", "user_proxy = autogen.UserProxyAgent(\n", " name=\"User_proxy\",\n", " system_message=\"A human admin.\",\n", @@ -409,7 +443,8 @@ "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", "administrator.start(\n", " task,\n", - " mg_config[0],\n", + " mg_llm_config,\n", + " gpt3_5_server_id,\n", " user_proxy=user_proxy,\n", " initiate_agent_name=\"user\"\n", ")\n", @@ -418,15 +453,15 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-29T09:37:45.902790700Z", - "start_time": "2023-10-29T09:36:44.499788900Z" + "end_time": "2023-10-29T19:38:08.180214600Z", + "start_time": "2023-10-29T19:37:50.542041500Z" } }, "id": "dc7e531b53e5d7d6" }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 3, "outputs": [ { "name": "stdout", @@ -439,7 +474,7 @@ "--------------------------------------------------------------------------------\n", "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the abstract or content of the paper to identify its potential applications in software.\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", "\n", "Here is the Python code to accomplish this task:\n", "\n", @@ -454,30 +489,34 @@ "# Parse the XML response and extract the paper details\n", "xml_data = response.text\n", "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "paper_abstract = xml_data.split(\"\")[1].split(\"\")[0]\n", + "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", "\n", "# Print the paper details\n", "print(\"Latest Paper on GPT-4:\")\n", "print(\"Title:\", paper_title)\n", - "print(\"Abstract:\", paper_abstract)\n", + "print(\"Summary:\", paper_summary)\n", + "\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", + "\n", "```\n", "\n", - "Please note that the above code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", "\n", - "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", + "When you run the code, it will output the latest paper on GPT-4 from arXiv, including the title and summary. It will also analyze the paper's summary to identify potential applications in software and print them.\n", "\n", - "Please let me know if you need any further assistance.\n", + "Please let me know if you need any further assistance!\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", "\n", - "I'm sorry for the confusion, but the code I provided is not executable because it's not in a code block. Here is the correct version:\n", - "\n", - "```python\n", - "# filename: arxiv_search.py\n", - "\n", + "# filename: gpt4_paper_search.py\n", "import requests\n", - "from bs4 import BeautifulSoup\n", + "import xml.etree.ElementTree as ET\n", "\n", "# Search for papers related to GPT-4 on arXiv\n", "search_query = \"GPT-4\"\n", @@ -485,35 +524,61 @@ "response = requests.get(api_url)\n", "\n", "# Parse the XML response and extract the paper details\n", - "soup = BeautifulSoup(response.content, 'html.parser')\n", - "entry = soup.find('entry')\n", - "\n", - "if entry:\n", - " paper_title = entry.title.text\n", - " paper_abstract = entry.summary.text\n", - "\n", - " # Print the paper details\n", - " print(\"Latest Paper on GPT-4:\")\n", - " print(\"Title:\", paper_title)\n", - " print(\"Abstract:\", paper_abstract)\n", - "else:\n", - " print(\"No papers found related to GPT-4.\")\n", - "```\n", + "root = ET.fromstring(response.content)\n", + "entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", + "paper_title = entry.find('{http://www.w3.org/2005/Atom}title').text\n", + "paper_summary = entry.find('{http://www.w3.org/2005/Atom}summary').text\n", "\n", - "Please note that the above code uses the `requests` and `BeautifulSoup` libraries to make an HTTP request to the arXiv API and parse the XML response. Make sure you have these libraries installed (`pip install requests beautifulsoup4`) before running the code.\n", + "# Print the paper details\n", + "print(\"Latest Paper on GPT-4:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Summary:\", paper_summary)\n", "\n", - "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", "\n", - "I apologize for the confusion, but the code I provided is not executable because it's not in a code block. Here is the correct version:\n", + "exitcode: 1 (execution failed)\n", + "Code output: \n", + "Traceback (most recent call last):\n", + " File \"\", line 10, in \n", + " paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", + "IndexError: list index out of range\n", "\n", - "```python\n", - "# filename: arxiv_search.py\n", "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\n", + "I apologize for the error. It seems that the code I provided to extract the paper details from the XML response is not working as expected. Let's try an alternative approach using the `xml.etree.ElementTree` module to parse the XML response.\n", + "\n", + "Please use the updated code below:\n", + "\n", + "```python\n", "import requests\n", - "from bs4 import BeautifulSoup\n", + "import xml.etree.ElementTree as ET\n", "\n", "# Search for papers related to GPT-4 on arXiv\n", "search_query = \"GPT-4\"\n", @@ -521,24 +586,27 @@ "response = requests.get(api_url)\n", "\n", "# Parse the XML response and extract the paper details\n", - "soup = BeautifulSoup(response.content, 'html.parser')\n", - "entry = soup.find('entry')\n", - "\n", - "if entry:\n", - " paper_title = entry.title.text\n", - " paper_abstract = entry.summary.text\n", - "\n", - " # Print the paper details\n", - " print(\"Latest Paper on GPT-4:\")\n", - " print(\"Title:\", paper_title)\n", - " print(\"Abstract:\", paper_abstract)\n", - "else:\n", - " print(\"No papers found related to GPT-4.\")\n", + "root = ET.fromstring(response.content)\n", + "entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", + "paper_title = entry.find('{http://www.w3.org/2005/Atom}title').text\n", + "paper_summary = entry.find('{http://www.w3.org/2005/Atom}summary').text\n", + "\n", + "# Print the paper details\n", + "print(\"Latest Paper on GPT-4:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Summary:\", paper_summary)\n", + "\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", "```\n", "\n", - "Please note that the above code uses the `requests` and `BeautifulSoup` libraries to make an HTTP request to the arXiv API and parse the XML response. Make sure you have these libraries installed (`pip install requests beautifulsoup4`) before running the code.\n", + "Please make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", "\n", - "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", + "Let me know if you encounter any further issues!\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", @@ -564,7 +632,7 @@ "Code output: \n", "Latest Paper on GPT-4:\n", "Title: Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models\n", - "Abstract: With LLMs shifting their role from statistical modeling of language to\n", + "Summary: With LLMs shifting their role from statistical modeling of language to\n", "serving as general-purpose AI agents, how should LLM evaluations change?\n", "Arguably, a key ability of an AI agent is to flexibly combine, as needed, the\n", "basic skills it has learned. The capability to combine skills plays an\n", @@ -591,21 +659,17 @@ "open evaluations for AI capabilities of future models.\n", "\n", "\n", + "Potential Applications in Software:\n", + "- Chatbots\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", "\n", - "Based on the abstract of the latest paper titled \"Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models\", the potential applications of GPT-4 in software can be inferred as follows:\n", - "\n", - "1. **General-Purpose AI Agents**: GPT-4 can serve as a general-purpose AI agent, which means it can be used in a wide range of software applications that require AI capabilities. This includes but is not limited to chatbots, virtual assistants, content generation tools, and more.\n", - "\n", - "2. **Combining Skills**: The paper introduces a new evaluation called Skill-Mix to measure the ability to combine skills. This suggests that GPT-4 can be used in software applications where multiple AI skills need to be combined. For example, a software application might require an AI to understand natural language, generate relevant responses, and also exhibit some level of reasoning or decision-making.\n", + "The latest paper on GPT-4 from arXiv is titled \"Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models\". The summary of the paper is as follows:\n", "\n", - "3. **Automatic Grading**: The paper discusses automatic grading of the results produced by GPT-4. This suggests potential applications in educational software or any other software that requires automatic grading or evaluation of text.\n", + "\"With LLMs shifting their role from statistical modeling of language to serving as general-purpose AI agents, how should LLM evaluations change? Arguably, a key ability of an AI agent is to flexibly combine, as needed, the basic skills it has learned. The capability to combine skills plays an important role in (human) pedagogy and also in a paper on emergence phenomena (Arora & Goyal, 2023). This work introduces Skill-Mix, a new evaluation to measure ability to combine skills. Using a list of $N$ skills the evaluator repeatedly picks random subsets of $k$ skills and asks the LLM to produce text combining that subset of skills. Since the number of subsets grows like $N^k$, for even modest $k$ this evaluation will, with high probability, require the LLM to produce text significantly different from any text in the training set. The paper develops a methodology for (a) designing and administering such an evaluation, and (b) automatic grading (plus spot-checking by humans) of the results using GPT-4 as well as the open LLaMA-2 70B model. Administering a version of to popular chatbots gave results that, while generally in line with prior expectations, contained surprises. Sizeable differences exist among model capabilities that are not captured by their ranking on popular LLM leaderboards (\"cramming for the leaderboard\"). Furthermore, simple probability calculations indicate that GPT-4's reasonable performance on $k=5$ is suggestive of going beyond \"stochastic parrot\" behavior (Bender et al., 2021), i.e., it combines skills in ways that it had not seen during training. We sketch how the methodology can lead to a Skill-Mix based eco-system of open evaluations for AI capabilities of future models.\"\n", "\n", - "4. **Beyond \"Stochastic Parrot\" Behavior**: The paper suggests that GPT-4's performance indicates it goes beyond simply repeating patterns it has seen during training. This implies that GPT-4 could be used in software applications that require novel text generation or creative problem-solving.\n", - "\n", - "Please note that these are potential applications inferred from the abstract of the paper. The actual applications could be more specific and varied depending on the detailed content of the paper and the specific requirements of the software.\n", + "Based on the summary, the potential application of GPT-4 in software identified is in the development of Chatbots. The paper discusses the use of GPT-4 in creating chatbots that can flexibly combine different skills to produce text that is significantly different from any text in the training set. This suggests that GPT-4 could be used to create more advanced and versatile chatbots that can handle a wider range of tasks and conversations.\n", "\n", "TERMINATE\n", "\n", @@ -644,24 +708,33 @@ "\n", "\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "\n", - "\n", "--------------------------------------------------------------------------------\n" ] } ], "source": [ "administrator = AgentCreator()\n", + "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", + "\n", + "# build endpoint; create agent; get config (manager); initialize user proxy; task start; clear agent\n", + "gpt3_5_config_list, gpt3_5_server_id = administrator.build_endpoint('gpt-3.5-turbo', config_path=config_path)\n", + "gpt4_config_list, gpt4_server_id = administrator.build_endpoint('gpt-4', config_path=config_path)\n", + "\n", + "coder_llm_config = {\n", + " 'config_list': gpt3_5_config_list,\n", + " 'temperature': 0,\n", + "}\n", + "pm_llm_config = {\n", + " 'config_list': gpt4_config_list,\n", + " 'temperature': 0,\n", + "}\n", + "mg_llm_config = {\n", + " 'config_list': gpt3_5_config_list,\n", + " 'temperature': 0,\n", + "}\n", "\n", - "# create agent; get config (manager); initialize user proxy; task start; clear agent\n", - "administrator.create_agent('Coder_gpt-3_5', 'gpt-3.5-turbo')\n", - "administrator.create_agent('Product_manager_gpt-4', 'gpt-4')\n", - "mg_config, _ = administrator.get_config('gpt-3.5-turbo', seed=41)\n", + "administrator.create_agent('Coder_gpt-3_5', coder_llm_config, gpt3_5_server_id)\n", + "administrator.create_agent('Product_manager_gpt-4', pm_llm_config, gpt4_server_id)\n", "user_proxy = autogen.UserProxyAgent(\n", " name=\"User_proxy\",\n", " system_message=\"A human admin.\",\n", @@ -672,7 +745,8 @@ "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", "administrator.start(\n", " task,\n", - " mg_config[0],\n", + " mg_llm_config,\n", + " gpt3_5_server_id,\n", " user_proxy=user_proxy,\n", " initiate_agent_name=\"user\"\n", ")\n", @@ -681,8 +755,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-29T09:48:33.383593800Z", - "start_time": "2023-10-29T09:48:32.580332400Z" + "end_time": "2023-10-29T19:41:44.526680Z", + "start_time": "2023-10-29T19:40:26.388225100Z" } }, "id": "7d52e3d9a1bf91cb" From d7725adfcf4a9523549275173673f5693fd09c5b Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Tue, 31 Oct 2023 15:20:00 +0900 Subject: [PATCH 05/74] update AgentCreator --- notebook/oai_create_agent.ipynb | 671 +++++++++++--------------------- 1 file changed, 231 insertions(+), 440 deletions(-) diff --git a/notebook/oai_create_agent.ipynb b/notebook/oai_create_agent.ipynb index 0ca441d9451..df8ad752df3 100644 --- a/notebook/oai_create_agent.ipynb +++ b/notebook/oai_create_agent.ipynb @@ -39,123 +39,164 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 9, "outputs": [], "source": [ "import autogen\n", - "import subprocess as sp\n", "import time\n", + "import socket\n", + "import subprocess as sp\n", "from typing import *" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-29T19:40:21.535921400Z", - "start_time": "2023-10-29T19:40:20.942693900Z" + "end_time": "2023-10-31T06:18:25.637679500Z", + "start_time": "2023-10-31T06:18:25.613603400Z" } }, "id": "8d06a99bd05026bb" }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 10, "outputs": [], "source": [ "class AgentCreator:\n", " \"\"\"\n", " Descriptions\n", " \"\"\"\n", - "\n", - " agent_procs: Dict[str, sp.Popen] = {}\n", - " agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {}\n", + " host: str\n", + " task: str\n", + " config_path: str\n", + " open_ports: List[str] = []\n", + " agent_procs: Dict[str, Tuple[sp.Popen, str]] = {}\n", " openai_server_name: str = 'openai'\n", + " endpoint_building_timeout: Optional[int]\n", + " agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {}\n", + " max_tokens: int = 945\n", + " \n", + " user_proxy: autogen.UserProxyAgent = None\n", + " group_chat_manager_config: dict = None\n", + " manager_system_message: str = 'Group chat manager.'\n", "\n", " def __init__(\n", " self,\n", + " task: str,\n", + " host: str = 'localhost',\n", + " config_path: str = 'OAI_CONFIG_LIST',\n", " endpoint_building_timeout: Optional[int] = 180\n", " ):\n", " \"\"\"\n", " Args:\n", + " task: description of a task.\n", " endpoint_building_timeout: timeout for building up an endpoint server.\n", + " config_path: path of the OpenAI api configs.\n", + " host: endpoint host.\n", " \"\"\"\n", + " self.task = task\n", " self.endpoint_building_timeout = endpoint_building_timeout\n", + " self.config_path = config_path\n", + " self.host = host\n", "\n", - " def build_endpoint(\n", + " print('Initializing usable port...')\n", + " for port in range(8000, 65535):\n", + " if self._is_port_open(host, port):\n", + " self.open_ports.append(str(port))\n", + " print(f'{len(self.open_ports)} ports found.')\n", + "\n", + " @staticmethod\n", + " def _is_port_open(host, port):\n", + " \"\"\"Check if a port is open.\"\"\"\n", + " try:\n", + " s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n", + " s.settimeout(10)\n", + " s.bind((host, int(port)))\n", + " s.close()\n", + " return True\n", + " except OSError:\n", + " return False\n", + " \n", + " def set_task(self, task: str):\n", + " self.task = task\n", + "\n", + " def create_agent(\n", " self,\n", + " agent_name: str,\n", " model_name_or_hf_repo: str,\n", - " world_size: Optional[int] = 1,\n", - " host: Optional[str] = \"127.0.0.1\",\n", - " port: Optional[str] = \"8000\",\n", - " config_path: Optional[str] = './OAI_CONFIG_LIST'\n", - " ) -> Tuple[dict, str]:\n", + " llm_config: dict,\n", + " system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE,\n", + " world_size: Optional[int] = 1\n", + " ) -> autogen.AssistantAgent:\n", " \"\"\"\n", " Descriptions\n", "\n", " Args:\n", - " model_name_or_hf_repo: model name (e.g., \"gpt-3.5-turbo\", \"gpt-4\") or huggingface repository\n", - " (e.g., \"meta-llama/Llama-2-7b-chat-hf\") use for starting an endpoint.\n", + " agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...)\n", + " model_name_or_hf_repo:\n", + " llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...).\n", + " system_message: system prompt use to format an agent's behavior.\n", " world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs).\n", - " host: the endpoint's API address.\n", - " port: the port of the API address the endpoint listen to.\n", - " config_path: config list path of OpenAI api.\n", "\n", " Returns:\n", - " (agent config list, endpoint server id)\n", + " agent: a set-up agent.\n", " \"\"\"\n", " config_list = autogen.config_list_from_json(\n", - " config_path,\n", + " self.config_path,\n", " filter_dict={\n", " 'model': [model_name_or_hf_repo]\n", " }\n", " )\n", - " if \"gpt-\" in model_name_or_hf_repo:\n", + " if 'gpt-' in model_name_or_hf_repo:\n", " server_id = self.openai_server_name\n", " else:\n", " model_name = model_name_or_hf_repo.split('/')[-1]\n", - " config_list[0]['api_base'] = f\"http://{host}:{port}/v1\"\n", - " server_id = f'{model_name}_{host}_{port}'\n", + " server_id = f'{model_name}_{self.host}'\n", " if self.agent_procs.get(server_id, None) is None:\n", + " while True:\n", + " port = self.open_ports.pop()\n", + " if self._is_port_open(self.host, port):\n", + " break\n", + "\n", " # Use vLLM to set up a server with OpenAI API support.\n", " agent_proc = sp.Popen(['python', '-m', 'vllm.entrypoints.openai.api_server',\n", - " f'--host', f'{host}',\n", + " f'--host', f'{self.host}',\n", " f'--port', f'{port}',\n", " f'--model', f'{model_name_or_hf_repo}',\n", " f'--tensor-parallel-size', f'{world_size}'], stdout=sp.PIPE, stderr=sp.STDOUT)\n", " timeout_start = time.time()\n", + "\n", " while True:\n", " server_stdout = agent_proc.stdout.readline()\n", + " if server_stdout != b'':\n", + " print(server_stdout)\n", " timeout_end = time.time()\n", " if b\"running\" in server_stdout:\n", - " print(f\"Running {model_name_or_hf_repo} on http://{host}:{port} with tensor parallel size {world_size}.\")\n", + " print(\n", + " f'Running {model_name_or_hf_repo} on http://{self.host}:{port} '\n", + " f'with tensor parallel size {world_size}.')\n", " break\n", + " elif b\"address already in use\" in server_stdout:\n", + " raise RuntimeError(f'{self.host}:{port} already in use. Fail to set up the endpoint for '\n", + " f'{model_name_or_hf_repo} on {self.host}:{port}.')\n", " elif timeout_end - timeout_start > self.endpoint_building_timeout:\n", " raise RuntimeError(f'Timeout exceed. Fail to set up the endpoint for '\n", - " f'{model_name_or_hf_repo} on {host}:{port}.')\n", - " self.agent_procs[server_id] = agent_proc\n", - "\n", - " return config_list, server_id\n", - "\n", - " def create_agent(\n", - " self,\n", - " agent_name: str,\n", - " llm_config: dict,\n", - " server_id: str,\n", - " **kwargs\n", - " ) -> autogen.AssistantAgent:\n", - " \"\"\"\n", - " Descriptions\n", + " f'{model_name_or_hf_repo} on {self.host}:{port}.')\n", + " self.agent_procs[server_id] = (agent_proc, port)\n", + " else:\n", + " port = self.agent_procs[server_id][1]\n", "\n", - " Args:\n", - " agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...)\n", - " llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...).\n", - " server_id: endpoint server id.\n", + " config_list[0]['api_base'] = f'http://{self.host}:{port}/v1'\n", "\n", - " Returns:\n", - " agent: a set-up agent.\n", - " \"\"\"\n", + " current_config = llm_config.copy()\n", + " current_config.update({\n", + " 'config_list': config_list,\n", + " 'model': model_name_or_hf_repo,\n", + " 'max_tokens': self.max_tokens\n", + " })\n", " agent = autogen.AssistantAgent(name=agent_name,\n", - " llm_config=llm_config,\n", - " **kwargs)\n", + " llm_config=current_config.copy(),\n", + " system_message=system_message)\n", " self.agent_procs_assign[agent_name] = (agent, server_id)\n", " return agent\n", "\n", @@ -181,8 +222,9 @@ " for _, iter_sid in self.agent_procs_assign.values():\n", " if server_id == iter_sid:\n", " return\n", - " self.agent_procs[server_id].terminate()\n", - " \n", + " self.agent_procs[server_id][0].terminate()\n", + " self.open_ports.append(server_id.split('_')[-1])\n", + "\n", " def clear_all(self):\n", " \"\"\"\n", " Clear all cached agents.\n", @@ -190,50 +232,73 @@ " for agent_name in [agent_name for agent_name in self.agent_procs_assign.keys()]:\n", " self.clear_agent(agent_name)\n", "\n", + " def build(\n", + " self,\n", + " code: bool = None\n", + " ):\n", + " # TODO: not completed.\n", + " # build endpoint; create agent; get config (manager); initialize user proxy; task start; clear agent\n", + " default_llm_config = {\n", + " 'temperature': 0,\n", + " }\n", + " \n", + " self.create_agent('Coder_gpt-35', 'gpt-3.5-turbo', default_llm_config)\n", + " self.create_agent('Product_manager_gpt-35', 'gpt-3.5-turbo', default_llm_config)\n", + " \n", + " self.user_proxy = autogen.UserProxyAgent(\n", + " name=\"User_proxy\",\n", + " system_message=\"A human admin.\",\n", + " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", + " human_input_mode=\"TERMINATE\"\n", + " )\n", + " \n", + " config_list = autogen.config_list_from_json(\n", + " self.config_path,\n", + " filter_dict={\n", + " 'model': ['gpt-4']\n", + " }\n", + " )\n", + " self.group_chat_manager_config = default_llm_config.copy()\n", + " self.group_chat_manager_config['config_list'] = config_list\n", + " self.manager_system_message = 'Group chat manager.'\n", + " \n", + "\n", " def start(\n", " self,\n", - " task: str,\n", - " manager_llm_config: dict,\n", - " manager_server_id: str,\n", " max_round: Optional[int] = 12,\n", " init_messages: Optional[List[dict]] = [],\n", - " user_proxy: autogen.UserProxyAgent = None,\n", - " initiate_agent_name: str = \"user\"\n", + " initiate_agent_name: str = 'user'\n", " ):\n", " \"\"\"\n", " Descriptions\n", "\n", " Args:\n", - " task: task description.\n", - " manager_llm_config: LLM configuration for a GroupChatManager.\n", - " manager_server_id: id of the server the manager depending on.\n", " max_round: the maximum number of rounds.\n", " init_messages: input messages before the task start. This can be the chat history from other group chat\n", " or some preliminary of the task.\n", - " user_proxy: a user proxy for human interaction.\n", " initiate_agent_name: the name of an agent use to initialize the group chat.\n", " \"\"\"\n", " agent_list = [agent for agent, _ in self.agent_procs_assign.values()]\n", - " if user_proxy is not None:\n", - " agent_list.append(user_proxy)\n", + " if self.user_proxy is not None:\n", + " agent_list.append(self.user_proxy)\n", " group_chat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round)\n", - " manager = autogen.GroupChatManager(groupchat=group_chat, llm_config=manager_llm_config)\n", - " self.agent_procs_assign['manager'] = (manager, manager_server_id)\n", "\n", - " if initiate_agent_name == \"user\" and user_proxy is not None:\n", - " user_proxy.initiate_chat(manager, message=task)\n", + " manager = autogen.GroupChatManager(groupchat=group_chat,\n", + " llm_config=self.group_chat_manager_config,\n", + " system_message=self.manager_system_message)\n", + "\n", + " if initiate_agent_name == \"user\" and self.user_proxy is not None:\n", + " self.user_proxy.initiate_chat(manager, message=self.task)\n", " else:\n", " for agent in agent_list:\n", " if initiate_agent_name == agent.name():\n", - " agent.initiate_chat(manager, message=task)\n", - " \n", - " self.clear_agent('manager')" + " agent.initiate_chat(manager, message=self.task)" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-29T19:40:21.588597Z", - "start_time": "2023-10-29T19:40:21.544449200Z" + "end_time": "2023-10-31T06:18:27.355162100Z", + "start_time": "2023-10-31T06:18:27.310913400Z" } }, "id": "d729233c9524f875" @@ -241,240 +306,31 @@ { "cell_type": "markdown", "source": [ - "# Test" + "# Test (GPT)" ], "metadata": { "collapsed": false }, - "id": "8f2c3f521de23faa" - }, - { - "cell_type": "code", - "execution_count": 3, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running facebook/opt-125m on http://127.0.0.1:8000 with tensor parallel size 1.\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", - "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", - "\n", - "Here is the Python code to accomplish this task:\n", - "\n", - "```python\n", - "import requests\n", - "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", - "\n", - "# Parse the XML response and extract the paper details\n", - "xml_data = response.text\n", - "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", - "\n", - "# Print the paper details\n", - "print(\"Latest Paper on GPT-4:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", - "\n", - "```\n", - "\n", - "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", - "\n", - "When you run the code, it will output the latest paper on GPT-4 from arXiv, including the title and summary. It will also analyze the paper's summary to identify potential applications in software and print them.\n", - "\n", - "Please let me know if you need any further assistance!\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_opt-125m\u001B[0m (to chat_manager):\n", - "\n", - " If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", - "\n", - "I apologize for the repeated message. If you have a specific problem that you would like to solve using Python code, please provide the details of the problem and I will be happy to assist you with a Python solution.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_opt-125m\u001B[0m (to chat_manager):\n", - "\n", - " If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code to solve the problem.\n", - "### Human: If you need to solve a problem with a python code, please use the python code\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "[autogen.oai.completion: 10-30 04:38:04] {238} INFO - retrying in 10 seconds...\n", - "Traceback (most recent call last):\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 403, in handle_error_response\n", - " error_data = resp[\"error\"]\n", - "KeyError: 'error'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/elpis_ubuntu/LLM/autogen/autogen/oai/completion.py\", line 224, in _get_response\n", - " response = openai_completion.create(request_timeout=request_timeout, **config)\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py\", line 25, in create\n", - " return super().create(*args, **kwargs)\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py\", line 153, in create\n", - " response, _, api_key = requestor.request(\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 298, in request\n", - " resp, got_stream = self._interpret_response(result, stream)\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 700, in _interpret_response\n", - " self._interpret_response_line(\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 763, in _interpret_response_line\n", - " raise self.handle_error_response(\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 405, in handle_error_response\n", - " raise error.APIError(\n", - "openai.error.APIError: Invalid response object from API: '{\"object\":\"error\",\"message\":\"This model\\'s maximum context length is 2048 tokens. However, you requested 2092 tokens (1892 in the messages, 200 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":null}' (HTTP response code was 400)\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mKeyError\u001B[0m Traceback (most recent call last)", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:403\u001B[0m, in \u001B[0;36mAPIRequestor.handle_error_response\u001B[0;34m(self, rbody, rcode, resp, rheaders, stream_error)\u001B[0m\n\u001B[1;32m 402\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 403\u001B[0m error_data \u001B[38;5;241m=\u001B[39m \u001B[43mresp\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43merror\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\n\u001B[1;32m 404\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\u001B[38;5;167;01mKeyError\u001B[39;00m, \u001B[38;5;167;01mTypeError\u001B[39;00m):\n", - "\u001B[0;31mKeyError\u001B[0m: 'error'", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001B[0;31mAPIError\u001B[0m Traceback (most recent call last)", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:224\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 223\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 224\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mopenai_completion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 225\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\n\u001B[1;32m 226\u001B[0m ServiceUnavailableError,\n\u001B[1;32m 227\u001B[0m APIConnectionError,\n\u001B[1;32m 228\u001B[0m ):\n\u001B[1;32m 229\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py:25\u001B[0m, in \u001B[0;36mChatCompletion.create\u001B[0;34m(cls, *args, **kwargs)\u001B[0m\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 25\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 26\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m TryAgain \u001B[38;5;28;01mas\u001B[39;00m e:\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001B[0m, in \u001B[0;36mEngineAPIResource.create\u001B[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001B[0m\n\u001B[1;32m 138\u001B[0m (\n\u001B[1;32m 139\u001B[0m deployment_id,\n\u001B[1;32m 140\u001B[0m engine,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 150\u001B[0m api_key, api_base, api_type, api_version, organization, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams\n\u001B[1;32m 151\u001B[0m )\n\u001B[0;32m--> 153\u001B[0m response, _, api_key \u001B[38;5;241m=\u001B[39m \u001B[43mrequestor\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 154\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mpost\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 155\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 156\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 157\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 158\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 159\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 160\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 161\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream:\n\u001B[1;32m 164\u001B[0m \u001B[38;5;66;03m# must be an iterator\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:298\u001B[0m, in \u001B[0;36mAPIRequestor.request\u001B[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 288\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mrequest_raw(\n\u001B[1;32m 289\u001B[0m method\u001B[38;5;241m.\u001B[39mlower(),\n\u001B[1;32m 290\u001B[0m url,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 296\u001B[0m request_timeout\u001B[38;5;241m=\u001B[39mrequest_timeout,\n\u001B[1;32m 297\u001B[0m )\n\u001B[0;32m--> 298\u001B[0m resp, got_stream \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_interpret_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mresult\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 299\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp, got_stream, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mapi_key\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:700\u001B[0m, in \u001B[0;36mAPIRequestor._interpret_response\u001B[0;34m(self, result, stream)\u001B[0m\n\u001B[1;32m 698\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 699\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m (\n\u001B[0;32m--> 700\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_interpret_response_line\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 701\u001B[0m \u001B[43m \u001B[49m\u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcontent\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdecode\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mutf-8\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 702\u001B[0m \u001B[43m \u001B[49m\u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstatus_code\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 703\u001B[0m \u001B[43m \u001B[49m\u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 704\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 705\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m,\n\u001B[1;32m 706\u001B[0m \u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[1;32m 707\u001B[0m )\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:763\u001B[0m, in \u001B[0;36mAPIRequestor._interpret_response_line\u001B[0;34m(self, rbody, rcode, rheaders, stream)\u001B[0m\n\u001B[1;32m 762\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream_error \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;241m200\u001B[39m \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m rcode \u001B[38;5;241m<\u001B[39m \u001B[38;5;241m300\u001B[39m:\n\u001B[0;32m--> 763\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhandle_error_response\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 764\u001B[0m \u001B[43m \u001B[49m\u001B[43mrbody\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrcode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mresp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrheaders\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mstream_error\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream_error\u001B[49m\n\u001B[1;32m 765\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 766\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:405\u001B[0m, in \u001B[0;36mAPIRequestor.handle_error_response\u001B[0;34m(self, rbody, rcode, resp, rheaders, stream_error)\u001B[0m\n\u001B[1;32m 404\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\u001B[38;5;167;01mKeyError\u001B[39;00m, \u001B[38;5;167;01mTypeError\u001B[39;00m):\n\u001B[0;32m--> 405\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m error\u001B[38;5;241m.\u001B[39mAPIError(\n\u001B[1;32m 406\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mInvalid response object from API: \u001B[39m\u001B[38;5;132;01m%r\u001B[39;00m\u001B[38;5;124m (HTTP response code \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 407\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwas \u001B[39m\u001B[38;5;132;01m%d\u001B[39;00m\u001B[38;5;124m)\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m (rbody, rcode),\n\u001B[1;32m 408\u001B[0m rbody,\n\u001B[1;32m 409\u001B[0m rcode,\n\u001B[1;32m 410\u001B[0m resp,\n\u001B[1;32m 411\u001B[0m )\n\u001B[1;32m 413\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124minternal_message\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m error_data:\n", - "\u001B[0;31mAPIError\u001B[0m: Invalid response object from API: '{\"object\":\"error\",\"message\":\"This model\\'s maximum context length is 2048 tokens. However, you requested 2092 tokens (1892 in the messages, 200 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":null}' (HTTP response code was 400)", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[3], line 32\u001B[0m\n\u001B[1;32m 24\u001B[0m user_proxy \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mUserProxyAgent(\n\u001B[1;32m 25\u001B[0m name\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mUser_proxy\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 26\u001B[0m system_message\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mA human admin.\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 27\u001B[0m code_execution_config\u001B[38;5;241m=\u001B[39m{\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mlast_n_messages\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;241m2\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mwork_dir\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mgroupchat\u001B[39m\u001B[38;5;124m\"\u001B[39m},\n\u001B[1;32m 28\u001B[0m human_input_mode\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mTERMINATE\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 29\u001B[0m )\n\u001B[1;32m 31\u001B[0m task \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m---> 32\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstart\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 33\u001B[0m \u001B[43m \u001B[49m\u001B[43mtask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 34\u001B[0m \u001B[43m \u001B[49m\u001B[43mmg_llm_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 35\u001B[0m \u001B[43m \u001B[49m\u001B[43mgpt3_5_server_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 36\u001B[0m \u001B[43m \u001B[49m\u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43muser_proxy\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 37\u001B[0m \u001B[43m \u001B[49m\u001B[43minitiate_agent_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43muser\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\n\u001B[1;32m 38\u001B[0m \u001B[43m)\u001B[49m\n\u001B[1;32m 39\u001B[0m administrator\u001B[38;5;241m.\u001B[39mclear_all()\n", - "Cell \u001B[0;32mIn[2], line 161\u001B[0m, in \u001B[0;36mAgentCreator.start\u001B[0;34m(self, task, manager_llm_config, manager_server_id, max_round, init_messages, user_proxy, initiate_agent_name)\u001B[0m\n\u001B[1;32m 158\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39magent_procs_assign[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmanager\u001B[39m\u001B[38;5;124m'\u001B[39m] \u001B[38;5;241m=\u001B[39m (manager, manager_server_id)\n\u001B[1;32m 160\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m initiate_agent_name \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124muser\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mand\u001B[39;00m user_proxy \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 161\u001B[0m \u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minitiate_chat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmanager\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 162\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m agent \u001B[38;5;129;01min\u001B[39;00m agent_list:\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:531\u001B[0m, in \u001B[0;36mConversableAgent.initiate_chat\u001B[0;34m(self, recipient, clear_history, silent, **context)\u001B[0m\n\u001B[1;32m 517\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"Initiate a chat with the recipient agent.\u001B[39;00m\n\u001B[1;32m 518\u001B[0m \n\u001B[1;32m 519\u001B[0m \u001B[38;5;124;03mReset the consecutive auto reply counter.\u001B[39;00m\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 528\u001B[0m \u001B[38;5;124;03m \"message\" needs to be provided if the `generate_init_message` method is not overridden.\u001B[39;00m\n\u001B[1;32m 529\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m 530\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_chat(recipient, clear_history)\n\u001B[0;32m--> 531\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_init_message\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mcontext\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrecipient\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:334\u001B[0m, in \u001B[0;36mConversableAgent.send\u001B[0;34m(self, message, recipient, request_reply, silent)\u001B[0m\n\u001B[1;32m 332\u001B[0m valid \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_append_oai_message(message, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124massistant\u001B[39m\u001B[38;5;124m\"\u001B[39m, recipient)\n\u001B[1;32m 333\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m valid:\n\u001B[0;32m--> 334\u001B[0m \u001B[43mrecipient\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreceive\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessage\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrequest_reply\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 335\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 336\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 337\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mMessage can\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mt be converted into a valid ChatCompletion message. Either content or function_call must be provided.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 338\u001B[0m )\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:462\u001B[0m, in \u001B[0;36mConversableAgent.receive\u001B[0;34m(self, message, sender, request_reply, silent)\u001B[0m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mreply_at_receive[sender] \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m:\n\u001B[1;32m 461\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m\n\u001B[0;32m--> 462\u001B[0m reply \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_reply\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mchat_messages\u001B[49m\u001B[43m[\u001B[49m\u001B[43msender\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 463\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 464\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msend(reply, sender, silent\u001B[38;5;241m=\u001B[39msilent)\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:781\u001B[0m, in \u001B[0;36mConversableAgent.generate_reply\u001B[0;34m(self, messages, sender, exclude)\u001B[0m\n\u001B[1;32m 779\u001B[0m \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m 780\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_match_trigger(reply_func_tuple[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtrigger\u001B[39m\u001B[38;5;124m\"\u001B[39m], sender):\n\u001B[0;32m--> 781\u001B[0m final, reply \u001B[38;5;241m=\u001B[39m \u001B[43mreply_func\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreply_func_tuple\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mconfig\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 782\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final:\n\u001B[1;32m 783\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m reply\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/groupchat.py:164\u001B[0m, in \u001B[0;36mGroupChatManager.run_chat\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 162\u001B[0m speaker \u001B[38;5;241m=\u001B[39m groupchat\u001B[38;5;241m.\u001B[39mselect_speaker(speaker, \u001B[38;5;28mself\u001B[39m)\n\u001B[1;32m 163\u001B[0m \u001B[38;5;66;03m# let the speaker speak\u001B[39;00m\n\u001B[0;32m--> 164\u001B[0m reply \u001B[38;5;241m=\u001B[39m \u001B[43mspeaker\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_reply\u001B[49m\u001B[43m(\u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 165\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mKeyboardInterrupt\u001B[39;00m:\n\u001B[1;32m 166\u001B[0m \u001B[38;5;66;03m# let the admin agent speak if interrupted\u001B[39;00m\n\u001B[1;32m 167\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m groupchat\u001B[38;5;241m.\u001B[39madmin_name \u001B[38;5;129;01min\u001B[39;00m groupchat\u001B[38;5;241m.\u001B[39magent_names:\n\u001B[1;32m 168\u001B[0m \u001B[38;5;66;03m# admin agent is one of the participants\u001B[39;00m\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:781\u001B[0m, in \u001B[0;36mConversableAgent.generate_reply\u001B[0;34m(self, messages, sender, exclude)\u001B[0m\n\u001B[1;32m 779\u001B[0m \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m 780\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_match_trigger(reply_func_tuple[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtrigger\u001B[39m\u001B[38;5;124m\"\u001B[39m], sender):\n\u001B[0;32m--> 781\u001B[0m final, reply \u001B[38;5;241m=\u001B[39m \u001B[43mreply_func\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreply_func_tuple\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mconfig\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 782\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final:\n\u001B[1;32m 783\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m reply\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:606\u001B[0m, in \u001B[0;36mConversableAgent.generate_oai_reply\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 603\u001B[0m messages \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_oai_messages[sender]\n\u001B[1;32m 605\u001B[0m \u001B[38;5;66;03m# TODO: #1143 handle token limit exceeded error\u001B[39;00m\n\u001B[0;32m--> 606\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43moai\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mChatCompletion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 607\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m-\u001B[39;49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpop\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontext\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_oai_system_message\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mllm_config\u001B[49m\n\u001B[1;32m 608\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 609\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m, oai\u001B[38;5;241m.\u001B[39mChatCompletion\u001B[38;5;241m.\u001B[39mextract_text_or_function_call(response)[\u001B[38;5;241m0\u001B[39m]\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:803\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 801\u001B[0m base_config[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmax_retry_period\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 802\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 803\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 804\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 805\u001B[0m \u001B[43m \u001B[49m\u001B[43muse_cache\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 806\u001B[0m \u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m<\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mlast\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01mor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 807\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mbase_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 808\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 809\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m response \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m 810\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m response\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:834\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 832\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m diskcache\u001B[38;5;241m.\u001B[39mCache(\u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mcache_path) \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39m_cache:\n\u001B[1;32m 833\u001B[0m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mset_cache(seed)\n\u001B[0;32m--> 834\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:239\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 237\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n\u001B[1;32m 238\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mretrying in \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mretry_wait_time\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m seconds...\u001B[39m\u001B[38;5;124m\"\u001B[39m, exc_info\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m)\n\u001B[0;32m--> 239\u001B[0m \u001B[43msleep\u001B[49m\u001B[43m(\u001B[49m\u001B[43mretry_wait_time\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 240\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (RateLimitError, Timeout) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[1;32m 241\u001B[0m time_left \u001B[38;5;241m=\u001B[39m max_retry_period \u001B[38;5;241m-\u001B[39m (time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m-\u001B[39m start_time \u001B[38;5;241m+\u001B[39m retry_wait_time)\n", - "\u001B[0;31mKeyboardInterrupt\u001B[0m: " - ] - } - ], - "source": [ - "administrator = AgentCreator()\n", - "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", - "\n", - "# build endpoint; create agent; get config (manager); initialize user proxy; task start; clear agent\n", - "gpt3_5_config_list, gpt3_5_server_id = administrator.build_endpoint('gpt-3.5-turbo', config_path=config_path)\n", - "opt_config_list, opt_server_id = administrator.build_endpoint('facebook/opt-125m', config_path=config_path)\n", - "\n", - "coder_llm_config = {\n", - " 'config_list': gpt3_5_config_list,\n", - " 'temperature': 0,\n", - "}\n", - "pm_llm_config = {\n", - " 'config_list': opt_config_list,\n", - " 'temperature': 0,\n", - " 'max_tokens': 200\n", - "}\n", - "mg_llm_config = {\n", - " 'config_list': gpt3_5_config_list,\n", - " 'temperature': 0,\n", - "}\n", - "\n", - "administrator.create_agent('Coder_gpt-3_5', coder_llm_config, gpt3_5_server_id)\n", - "administrator.create_agent('Product_manager_opt-125m', pm_llm_config, opt_server_id)\n", - "\n", - "user_proxy = autogen.UserProxyAgent(\n", - " name=\"User_proxy\",\n", - " system_message=\"A human admin.\",\n", - " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", - " human_input_mode=\"TERMINATE\"\n", - ")\n", - "\n", - "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", - "administrator.start(\n", - " task,\n", - " mg_llm_config,\n", - " gpt3_5_server_id,\n", - " user_proxy=user_proxy,\n", - " initiate_agent_name=\"user\"\n", - ")\n", - "administrator.clear_all()\n" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-10-29T19:38:08.180214600Z", - "start_time": "2023-10-29T19:37:50.542041500Z" - } - }, - "id": "dc7e531b53e5d7d6" + "id": "3d4b09d29b740bf7" }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 11, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Initializing usable port...\n", + "57522 ports found.\n", "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the abstract or content of the paper to identify its potential applications in software.\n", "\n", "Here is the Python code to accomplish this task:\n", "\n", @@ -489,57 +345,19 @@ "# Parse the XML response and extract the paper details\n", "xml_data = response.text\n", "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", + "paper_abstract = xml_data.split(\"\")[1].split(\"\")[0]\n", "\n", "# Print the paper details\n", "print(\"Latest Paper on GPT-4:\")\n", "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", - "\n", + "print(\"Abstract:\", paper_abstract)\n", "```\n", "\n", - "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "Please note that the above code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", "\n", - "When you run the code, it will output the latest paper on GPT-4 from arXiv, including the title and summary. It will also analyze the paper's summary to identify potential applications in software and print them.\n", + "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", "\n", - "Please let me know if you need any further assistance!\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", - "\n", - "# filename: gpt4_paper_search.py\n", - "import requests\n", - "import xml.etree.ElementTree as ET\n", - "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", - "\n", - "# Parse the XML response and extract the paper details\n", - "root = ET.fromstring(response.content)\n", - "entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", - "paper_title = entry.find('{http://www.w3.org/2005/Atom}title').text\n", - "paper_summary = entry.find('{http://www.w3.org/2005/Atom}summary').text\n", - "\n", - "# Print the paper details\n", - "print(\"Latest Paper on GPT-4:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", + "Please let me know if you need any further assistance.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", @@ -570,43 +388,47 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-3_5\u001B[0m (to chat_manager):\n", + "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", "\n", - "I apologize for the error. It seems that the code I provided to extract the paper details from the XML response is not working as expected. Let's try an alternative approach using the `xml.etree.ElementTree` module to parse the XML response.\n", + "I apologize for the error. It seems that the code failed to extract the paper details from the XML response. This could be due to changes in the XML structure or the absence of the expected tags.\n", "\n", - "Please use the updated code below:\n", + "To resolve this issue, we can use the `feedparser` library to parse the XML response and extract the paper details. Here's an updated version of the code:\n", "\n", "```python\n", "import requests\n", - "import xml.etree.ElementTree as ET\n", + "import feedparser\n", "\n", "# Search for papers related to GPT-4 on arXiv\n", "search_query = \"GPT-4\"\n", "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", "response = requests.get(api_url)\n", "\n", - "# Parse the XML response and extract the paper details\n", - "root = ET.fromstring(response.content)\n", - "entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", - "paper_title = entry.find('{http://www.w3.org/2005/Atom}title').text\n", - "paper_summary = entry.find('{http://www.w3.org/2005/Atom}summary').text\n", + "# Parse the XML response using feedparser\n", + "feed = feedparser.parse(response.text)\n", + "\n", + "# Extract the paper details\n", + "if len(feed.entries) > 0:\n", + " paper_title = feed.entries[0].title\n", + " paper_abstract = feed.entries[0].summary\n", + " paper_link = feed.entries[0].link\n", + " paper_published = feed.entries[0].published\n", + "\n", + " # Print the paper details\n", + " print(\"Latest Paper on GPT-4:\")\n", + " print(\"Title:\", paper_title)\n", + " print(\"Abstract:\", paper_abstract)\n", + " print(\"Link:\", paper_link)\n", + " print(\"Published:\", paper_published)\n", + "else:\n", + " print(\"No papers found on GPT-4.\")\n", "\n", - "# Print the paper details\n", - "print(\"Latest Paper on GPT-4:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", "```\n", "\n", - "Please make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "Please make sure you have the `requests` and `feedparser` libraries installed (`pip install requests feedparser`) before running the code.\n", + "\n", + "After executing the code, it will print the title, abstract, link, and published date of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", "\n", - "Let me know if you encounter any further issues!\n", + "Let me know if you need any further assistance.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", @@ -631,68 +453,52 @@ "exitcode: 0 (execution succeeded)\n", "Code output: \n", "Latest Paper on GPT-4:\n", - "Title: Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models\n", - "Summary: With LLMs shifting their role from statistical modeling of language to\n", - "serving as general-purpose AI agents, how should LLM evaluations change?\n", - "Arguably, a key ability of an AI agent is to flexibly combine, as needed, the\n", - "basic skills it has learned. The capability to combine skills plays an\n", - "important role in (human) pedagogy and also in a paper on emergence phenomena\n", - "(Arora & Goyal, 2023).\n", - " This work introduces Skill-Mix, a new evaluation to measure ability to\n", - "combine skills. Using a list of $N$ skills the evaluator repeatedly picks\n", - "random subsets of $k$ skills and asks the LLM to produce text combining that\n", - "subset of skills. Since the number of subsets grows like $N^k$, for even modest\n", - "$k$ this evaluation will, with high probability, require the LLM to produce\n", - "text significantly different from any text in the training set. The paper\n", - "develops a methodology for (a) designing and administering such an evaluation,\n", - "and (b) automatic grading (plus spot-checking by humans) of the results using\n", - "GPT-4 as well as the open LLaMA-2 70B model.\n", - " Administering a version of to popular chatbots gave results that, while\n", - "generally in line with prior expectations, contained surprises. Sizeable\n", - "differences exist among model capabilities that are not captured by their\n", - "ranking on popular LLM leaderboards (\"cramming for the leaderboard\").\n", - "Furthermore, simple probability calculations indicate that GPT-4's reasonable\n", - "performance on $k=5$ is suggestive of going beyond \"stochastic parrot\" behavior\n", - "(Bender et al., 2021), i.e., it combines skills in ways that it had not seen\n", - "during training.\n", - " We sketch how the methodology can lead to a Skill-Mix based eco-system of\n", - "open evaluations for AI capabilities of future models.\n", - "\n", - "\n", - "Potential Applications in Software:\n", - "- Chatbots\n", + "Title: Building Real-World Meeting Summarization Systems using Large Language\n", + " Models: A Practical Perspective\n", + "Abstract: This paper studies how to effectively build meeting summarization systems for\n", + "real-world usage using large language models (LLMs). For this purpose, we\n", + "conduct an extensive evaluation and comparison of various closed-source and\n", + "open-source LLMs, namely, GPT-4, GPT- 3.5, PaLM-2, and LLaMA-2. Our findings\n", + "reveal that most closed-source LLMs are generally better in terms of\n", + "performance. However, much smaller open-source models like LLaMA- 2 (7B and\n", + "13B) could still achieve performance comparable to the large closed-source\n", + "models even in zero-shot scenarios. Considering the privacy concerns of\n", + "closed-source models for only being accessible via API, alongside the high cost\n", + "associated with using fine-tuned versions of the closed-source models, the\n", + "opensource models that can achieve competitive performance are more\n", + "advantageous for industrial use. Balancing performance with associated costs\n", + "and privacy concerns, the LLaMA-2-7B model looks more promising for industrial\n", + "usage. In sum, this paper offers practical insights on using LLMs for\n", + "real-world business meeting summarization, shedding light on the trade-offs\n", + "between performance and cost.\n", + "Link: http://arxiv.org/abs/2310.19233v1\n", + "Published: 2023-10-30T02:25:21Z\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_gpt-4\u001B[0m (to chat_manager):\n", + "\u001B[33mProduct_manager_gpt-35\u001B[0m (to chat_manager):\n", "\n", - "The latest paper on GPT-4 from arXiv is titled \"Skill-Mix: a Flexible and Expandable Family of Evaluations for AI models\". The summary of the paper is as follows:\n", + "Based on the latest paper titled \"Building Real-World Meeting Summarization Systems using Large Language Models: A Practical Perspective\" on GPT-4, here are some potential applications of GPT-4 in software:\n", "\n", - "\"With LLMs shifting their role from statistical modeling of language to serving as general-purpose AI agents, how should LLM evaluations change? Arguably, a key ability of an AI agent is to flexibly combine, as needed, the basic skills it has learned. The capability to combine skills plays an important role in (human) pedagogy and also in a paper on emergence phenomena (Arora & Goyal, 2023). This work introduces Skill-Mix, a new evaluation to measure ability to combine skills. Using a list of $N$ skills the evaluator repeatedly picks random subsets of $k$ skills and asks the LLM to produce text combining that subset of skills. Since the number of subsets grows like $N^k$, for even modest $k$ this evaluation will, with high probability, require the LLM to produce text significantly different from any text in the training set. The paper develops a methodology for (a) designing and administering such an evaluation, and (b) automatic grading (plus spot-checking by humans) of the results using GPT-4 as well as the open LLaMA-2 70B model. Administering a version of to popular chatbots gave results that, while generally in line with prior expectations, contained surprises. Sizeable differences exist among model capabilities that are not captured by their ranking on popular LLM leaderboards (\"cramming for the leaderboard\"). Furthermore, simple probability calculations indicate that GPT-4's reasonable performance on $k=5$ is suggestive of going beyond \"stochastic parrot\" behavior (Bender et al., 2021), i.e., it combines skills in ways that it had not seen during training. We sketch how the methodology can lead to a Skill-Mix based eco-system of open evaluations for AI capabilities of future models.\"\n", + "1. Meeting Summarization: The paper specifically focuses on building meeting summarization systems using large language models like GPT-4. GPT-4 can be used to automatically generate summaries of business meetings, saving time and effort for participants.\n", "\n", - "Based on the summary, the potential application of GPT-4 in software identified is in the development of Chatbots. The paper discusses the use of GPT-4 in creating chatbots that can flexibly combine different skills to produce text that is significantly different from any text in the training set. This suggests that GPT-4 could be used to create more advanced and versatile chatbots that can handle a wider range of tasks and conversations.\n", + "2. Natural Language Processing (NLP): GPT-4, being a large language model, can be used for various NLP tasks such as text generation, sentiment analysis, question answering, and language translation. It can enhance the capabilities of software applications that require understanding and generating human-like text.\n", "\n", - "TERMINATE\n", + "3. Chatbots and Virtual Assistants: GPT-4 can be used to power chatbots and virtual assistants, enabling more natural and context-aware conversations. It can improve the user experience by providing accurate and relevant responses to user queries.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "4. Content Generation: GPT-4 can be used to generate content for various purposes, such as writing articles, creating product descriptions, or generating code snippets. It can assist software developers and content creators in generating high-quality content efficiently.\n", "\n", + "5. Language Modeling and Understanding: GPT-4 can be used to improve language modeling and understanding in software applications. It can help in tasks like text completion, grammar correction, and language understanding, making software more intelligent and user-friendly.\n", "\n", + "These are just a few potential applications of GPT-4 in software. The actual applications may vary depending on the specific use case and requirements.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "\n", + "Let me know if there's anything else I can help you with.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", "\n", + "That's a great list of potential applications of GPT-4 in software! You've covered some key areas where GPT-4 can be leveraged to enhance software capabilities. These applications can greatly benefit businesses and users by automating tasks, improving user experiences, and generating high-quality content.\n", "\n", + "If you have any more questions or need further assistance, feel free to ask.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", @@ -702,61 +508,46 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "\n", + "\u001B[33mProduct_manager_gpt-35\u001B[0m (to chat_manager):\n", + "TERMINATE\n", "\n", - "--------------------------------------------------------------------------------\n" + "--------------------------------------------------------------------------------\n", + "[autogen.oai.completion: 10-31 15:18:49] {252} INFO - retrying in 10 seconds...\n", + "Traceback (most recent call last):\n", + " File \"/home/elpis_ubuntu/LLM/autogen/autogen/oai/completion.py\", line 224, in _get_response\n", + " response = openai_completion.create(request_timeout=request_timeout, **config)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py\", line 25, in create\n", + " return super().create(*args, **kwargs)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py\", line 153, in create\n", + " response, _, api_key = requestor.request(\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 298, in request\n", + " resp, got_stream = self._interpret_response(result, stream)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 700, in _interpret_response\n", + " self._interpret_response_line(\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 763, in _interpret_response_line\n", + " raise self.handle_error_response(\n", + "openai.error.RateLimitError: Rate limit reached for gpt-4 in organization org-sbI6D5OYthpTDqe4nmNmzQoS on tokens per min. Limit: 10000 / min. Please try again in 6ms. Visit https://platform.openai.com/account/rate-limits to learn more.\n" ] } ], "source": [ - "administrator = AgentCreator()\n", "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", "\n", - "# build endpoint; create agent; get config (manager); initialize user proxy; task start; clear agent\n", - "gpt3_5_config_list, gpt3_5_server_id = administrator.build_endpoint('gpt-3.5-turbo', config_path=config_path)\n", - "gpt4_config_list, gpt4_server_id = administrator.build_endpoint('gpt-4', config_path=config_path)\n", - "\n", - "coder_llm_config = {\n", - " 'config_list': gpt3_5_config_list,\n", - " 'temperature': 0,\n", - "}\n", - "pm_llm_config = {\n", - " 'config_list': gpt4_config_list,\n", - " 'temperature': 0,\n", - "}\n", - "mg_llm_config = {\n", - " 'config_list': gpt3_5_config_list,\n", - " 'temperature': 0,\n", - "}\n", - "\n", - "administrator.create_agent('Coder_gpt-3_5', coder_llm_config, gpt3_5_server_id)\n", - "administrator.create_agent('Product_manager_gpt-4', pm_llm_config, gpt4_server_id)\n", - "user_proxy = autogen.UserProxyAgent(\n", - " name=\"User_proxy\",\n", - " system_message=\"A human admin.\",\n", - " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", - " human_input_mode=\"TERMINATE\"\n", + "administrator = AgentCreator(\n", + " task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\",\n", + " config_path=config_path\n", ")\n", - "\n", - "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", + "administrator.build()\n", "administrator.start(\n", - " task,\n", - " mg_llm_config,\n", - " gpt3_5_server_id,\n", - " user_proxy=user_proxy,\n", " initiate_agent_name=\"user\"\n", ")\n", - "administrator.clear_all()" + "administrator.clear_all()\n" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-29T19:41:44.526680Z", - "start_time": "2023-10-29T19:40:26.388225100Z" + "end_time": "2023-10-31T06:19:27.389747500Z", + "start_time": "2023-10-31T06:18:28.729808500Z" } }, "id": "7d52e3d9a1bf91cb" From 8c767b509c9d873aaffdb6dbee7e30f17310448e Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Tue, 31 Oct 2023 16:51:48 +0900 Subject: [PATCH 06/74] update AgentCreator (modify params of build) --- notebook/oai_create_agent.ipynb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/notebook/oai_create_agent.ipynb b/notebook/oai_create_agent.ipynb index df8ad752df3..35b2bf25ca0 100644 --- a/notebook/oai_create_agent.ipynb +++ b/notebook/oai_create_agent.ipynb @@ -234,13 +234,11 @@ "\n", " def build(\n", " self,\n", + " default_llm_config: dict,\n", " code: bool = None\n", " ):\n", " # TODO: not completed.\n", " # build endpoint; create agent; get config (manager); initialize user proxy; task start; clear agent\n", - " default_llm_config = {\n", - " 'temperature': 0,\n", - " }\n", " \n", " self.create_agent('Coder_gpt-35', 'gpt-3.5-turbo', default_llm_config)\n", " self.create_agent('Product_manager_gpt-35', 'gpt-3.5-turbo', default_llm_config)\n", @@ -532,12 +530,15 @@ ], "source": [ "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", + "default_llm_config = {\n", + " 'temperature': 0\n", + "}\n", "\n", "administrator = AgentCreator(\n", " task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\",\n", " config_path=config_path\n", ")\n", - "administrator.build()\n", + "administrator.build(default_llm_config)\n", "administrator.start(\n", " initiate_agent_name=\"user\"\n", ")\n", From 92118705ef3ce9cb0cff9564e9c429c061ed7d36 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 3 Nov 2023 22:52:30 +0900 Subject: [PATCH 07/74] [update AgentCreator] add auto judgement of coding --- notebook/oai_create_agent.ipynb | 256 +++++++++++--------------------- 1 file changed, 85 insertions(+), 171 deletions(-) diff --git a/notebook/oai_create_agent.ipynb b/notebook/oai_create_agent.ipynb index 35b2bf25ca0..e7e6885e742 100644 --- a/notebook/oai_create_agent.ipynb +++ b/notebook/oai_create_agent.ipynb @@ -39,7 +39,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 27, "outputs": [], "source": [ "import autogen\n", @@ -51,15 +51,15 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-31T06:18:25.637679500Z", - "start_time": "2023-10-31T06:18:25.613603400Z" + "end_time": "2023-11-03T13:25:24.206940Z", + "start_time": "2023-11-03T13:25:24.205018900Z" } }, "id": "8d06a99bd05026bb" }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 32, "outputs": [], "source": [ "class AgentCreator:\n", @@ -75,11 +75,20 @@ " endpoint_building_timeout: Optional[int]\n", " agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {}\n", " max_tokens: int = 945\n", - " \n", + "\n", " user_proxy: autogen.UserProxyAgent = None\n", " group_chat_manager_config: dict = None\n", + " initiate_agent_name: str = 'user'\n", " manager_system_message: str = 'Group chat manager.'\n", "\n", + " CODING_PROMPT: str = '''Does the following task need programming \n", + " (i.e., access external API or tool by coding) to solve?\n", + "\n", + " TASK: {task}\n", + "\n", + " Answer only YES or NO.\n", + " '''\n", + "\n", " def __init__(\n", " self,\n", " task: str,\n", @@ -116,7 +125,7 @@ " return True\n", " except OSError:\n", " return False\n", - " \n", + "\n", " def set_task(self, task: str):\n", " self.task = task\n", "\n", @@ -235,37 +244,45 @@ " def build(\n", " self,\n", " default_llm_config: dict,\n", - " code: bool = None\n", + " coding: bool = None\n", " ):\n", " # TODO: not completed.\n", - " # build endpoint; create agent; get config (manager); initialize user proxy; task start; clear agent\n", - " \n", - " self.create_agent('Coder_gpt-35', 'gpt-3.5-turbo', default_llm_config)\n", - " self.create_agent('Product_manager_gpt-35', 'gpt-3.5-turbo', default_llm_config)\n", - " \n", - " self.user_proxy = autogen.UserProxyAgent(\n", - " name=\"User_proxy\",\n", - " system_message=\"A human admin.\",\n", - " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", - " human_input_mode=\"TERMINATE\"\n", - " )\n", - " \n", " config_list = autogen.config_list_from_json(\n", " self.config_path,\n", " filter_dict={\n", " 'model': ['gpt-4']\n", " }\n", " )\n", + " agent_configs = [('Coder_gpt-35', 'gpt-3.5-turbo'), ('Product_manager', 'gpt-3.5-turbo')]\n", + " \n", + " for agent_config in agent_configs:\n", + " self.create_agent(agent_config[0], agent_config[1], default_llm_config)\n", + " \n", + " if coding is None:\n", + " resp = autogen.ChatCompletion.create(\n", + " messages=[{\"role\": \"user\", \"content\": self.CODING_PROMPT.format(task=self.task)}],\n", + " **config_list[0]\n", + " ).choices[0].message.content\n", + " coding = True if resp == 'YES' else False\n", + "\n", + " if coding is True:\n", + " self.user_proxy = autogen.UserProxyAgent(\n", + " name=\"User_proxy\",\n", + " system_message=\"A human admin.\",\n", + " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", + " human_input_mode=\"TERMINATE\"\n", + " )\n", + " else:\n", + " self.initiate_agent_name = agent_configs[0][0]\n", + " \n", " self.group_chat_manager_config = default_llm_config.copy()\n", " self.group_chat_manager_config['config_list'] = config_list\n", " self.manager_system_message = 'Group chat manager.'\n", - " \n", "\n", " def start(\n", " self,\n", " max_round: Optional[int] = 12,\n", - " init_messages: Optional[List[dict]] = [],\n", - " initiate_agent_name: str = 'user'\n", + " init_messages: Optional[List[dict]] = []\n", " ):\n", " \"\"\"\n", " Descriptions\n", @@ -285,18 +302,18 @@ " llm_config=self.group_chat_manager_config,\n", " system_message=self.manager_system_message)\n", "\n", - " if initiate_agent_name == \"user\" and self.user_proxy is not None:\n", + " if self.initiate_agent_name == \"user\" and self.user_proxy is not None:\n", " self.user_proxy.initiate_chat(manager, message=self.task)\n", " else:\n", " for agent in agent_list:\n", - " if initiate_agent_name == agent.name():\n", + " if self.initiate_agent_name == agent.name():\n", " agent.initiate_chat(manager, message=self.task)" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-31T06:18:27.355162100Z", - "start_time": "2023-10-31T06:18:27.310913400Z" + "end_time": "2023-11-03T13:51:43.383138700Z", + "start_time": "2023-11-03T13:51:43.367423200Z" } }, "id": "d729233c9524f875" @@ -313,16 +330,15 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 34, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initializing usable port...\n", - "57522 ports found.\n", + "115036 ports found.\n", "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", @@ -385,146 +401,46 @@ "IndexError: list index out of range\n", "\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", - "\n", - "I apologize for the error. It seems that the code failed to extract the paper details from the XML response. This could be due to changes in the XML structure or the absence of the expected tags.\n", - "\n", - "To resolve this issue, we can use the `feedparser` library to parse the XML response and extract the paper details. Here's an updated version of the code:\n", - "\n", - "```python\n", - "import requests\n", - "import feedparser\n", - "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", - "\n", - "# Parse the XML response using feedparser\n", - "feed = feedparser.parse(response.text)\n", - "\n", - "# Extract the paper details\n", - "if len(feed.entries) > 0:\n", - " paper_title = feed.entries[0].title\n", - " paper_abstract = feed.entries[0].summary\n", - " paper_link = feed.entries[0].link\n", - " paper_published = feed.entries[0].published\n", - "\n", - " # Print the paper details\n", - " print(\"Latest Paper on GPT-4:\")\n", - " print(\"Title:\", paper_title)\n", - " print(\"Abstract:\", paper_abstract)\n", - " print(\"Link:\", paper_link)\n", - " print(\"Published:\", paper_published)\n", - "else:\n", - " print(\"No papers found on GPT-4.\")\n", - "\n", - "```\n", - "\n", - "Please make sure you have the `requests` and `feedparser` libraries installed (`pip install requests feedparser`) before running the code.\n", - "\n", - "After executing the code, it will print the title, abstract, link, and published date of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", - "\n", - "Let me know if you need any further assistance.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + "--------------------------------------------------------------------------------\n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "exitcode: 0 (execution succeeded)\n", - "Code output: \n", - "Latest Paper on GPT-4:\n", - "Title: Building Real-World Meeting Summarization Systems using Large Language\n", - " Models: A Practical Perspective\n", - "Abstract: This paper studies how to effectively build meeting summarization systems for\n", - "real-world usage using large language models (LLMs). For this purpose, we\n", - "conduct an extensive evaluation and comparison of various closed-source and\n", - "open-source LLMs, namely, GPT-4, GPT- 3.5, PaLM-2, and LLaMA-2. Our findings\n", - "reveal that most closed-source LLMs are generally better in terms of\n", - "performance. However, much smaller open-source models like LLaMA- 2 (7B and\n", - "13B) could still achieve performance comparable to the large closed-source\n", - "models even in zero-shot scenarios. Considering the privacy concerns of\n", - "closed-source models for only being accessible via API, alongside the high cost\n", - "associated with using fine-tuned versions of the closed-source models, the\n", - "opensource models that can achieve competitive performance are more\n", - "advantageous for industrial use. Balancing performance with associated costs\n", - "and privacy concerns, the LLaMA-2-7B model looks more promising for industrial\n", - "usage. In sum, this paper offers practical insights on using LLMs for\n", - "real-world business meeting summarization, shedding light on the trade-offs\n", - "between performance and cost.\n", - "Link: http://arxiv.org/abs/2310.19233v1\n", - "Published: 2023-10-30T02:25:21Z\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_gpt-35\u001B[0m (to chat_manager):\n", - "\n", - "Based on the latest paper titled \"Building Real-World Meeting Summarization Systems using Large Language Models: A Practical Perspective\" on GPT-4, here are some potential applications of GPT-4 in software:\n", - "\n", - "1. Meeting Summarization: The paper specifically focuses on building meeting summarization systems using large language models like GPT-4. GPT-4 can be used to automatically generate summaries of business meetings, saving time and effort for participants.\n", - "\n", - "2. Natural Language Processing (NLP): GPT-4, being a large language model, can be used for various NLP tasks such as text generation, sentiment analysis, question answering, and language translation. It can enhance the capabilities of software applications that require understanding and generating human-like text.\n", - "\n", - "3. Chatbots and Virtual Assistants: GPT-4 can be used to power chatbots and virtual assistants, enabling more natural and context-aware conversations. It can improve the user experience by providing accurate and relevant responses to user queries.\n", - "\n", - "4. Content Generation: GPT-4 can be used to generate content for various purposes, such as writing articles, creating product descriptions, or generating code snippets. It can assist software developers and content creators in generating high-quality content efficiently.\n", - "\n", - "5. Language Modeling and Understanding: GPT-4 can be used to improve language modeling and understanding in software applications. It can help in tasks like text completion, grammar correction, and language understanding, making software more intelligent and user-friendly.\n", - "\n", - "These are just a few potential applications of GPT-4 in software. The actual applications may vary depending on the specific use case and requirements.\n", - "\n", - "Let me know if there's anything else I can help you with.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", - "\n", - "That's a great list of potential applications of GPT-4 in software! You've covered some key areas where GPT-4 can be leveraged to enhance software capabilities. These applications can greatly benefit businesses and users by automating tasks, improving user experiences, and generating high-quality content.\n", - "\n", - "If you have any more questions or need further assistance, feel free to ask.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager_gpt-35\u001B[0m (to chat_manager):\n", - "TERMINATE\n", - "\n", - "--------------------------------------------------------------------------------\n", - "[autogen.oai.completion: 10-31 15:18:49] {252} INFO - retrying in 10 seconds...\n", - "Traceback (most recent call last):\n", - " File \"/home/elpis_ubuntu/LLM/autogen/autogen/oai/completion.py\", line 224, in _get_response\n", - " response = openai_completion.create(request_timeout=request_timeout, **config)\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py\", line 25, in create\n", - " return super().create(*args, **kwargs)\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py\", line 153, in create\n", - " response, _, api_key = requestor.request(\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 298, in request\n", - " resp, got_stream = self._interpret_response(result, stream)\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 700, in _interpret_response\n", - " self._interpret_response_line(\n", - " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py\", line 763, in _interpret_response_line\n", - " raise self.handle_error_response(\n", - "openai.error.RateLimitError: Rate limit reached for gpt-4 in organization org-sbI6D5OYthpTDqe4nmNmzQoS on tokens per min. Limit: 10000 / min. Please try again in 6ms. Visit https://platform.openai.com/account/rate-limits to learn more.\n" + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", + "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[34], line 11\u001B[0m\n\u001B[1;32m 6\u001B[0m administrator \u001B[38;5;241m=\u001B[39m AgentCreator(\n\u001B[1;32m 7\u001B[0m task\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 8\u001B[0m config_path\u001B[38;5;241m=\u001B[39mconfig_path\n\u001B[1;32m 9\u001B[0m )\n\u001B[1;32m 10\u001B[0m administrator\u001B[38;5;241m.\u001B[39mbuild(default_llm_config)\n\u001B[0;32m---> 11\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstart\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 12\u001B[0m administrator\u001B[38;5;241m.\u001B[39mclear_all()\n", + "Cell \u001B[0;32mIn[32], line 242\u001B[0m, in \u001B[0;36mAgentCreator.start\u001B[0;34m(self, max_round, init_messages)\u001B[0m\n\u001B[1;32m 237\u001B[0m manager \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mGroupChatManager(groupchat\u001B[38;5;241m=\u001B[39mgroup_chat,\n\u001B[1;32m 238\u001B[0m llm_config\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mgroup_chat_manager_config,\n\u001B[1;32m 239\u001B[0m system_message\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmanager_system_message)\n\u001B[1;32m 241\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39minitiate_agent_name \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124muser\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39muser_proxy \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 242\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minitiate_chat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmanager\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 243\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 244\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m agent \u001B[38;5;129;01min\u001B[39;00m agent_list:\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:531\u001B[0m, in \u001B[0;36mConversableAgent.initiate_chat\u001B[0;34m(self, recipient, clear_history, silent, **context)\u001B[0m\n\u001B[1;32m 517\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"Initiate a chat with the recipient agent.\u001B[39;00m\n\u001B[1;32m 518\u001B[0m \n\u001B[1;32m 519\u001B[0m \u001B[38;5;124;03mReset the consecutive auto reply counter.\u001B[39;00m\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 528\u001B[0m \u001B[38;5;124;03m \"message\" needs to be provided if the `generate_init_message` method is not overridden.\u001B[39;00m\n\u001B[1;32m 529\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m 530\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_chat(recipient, clear_history)\n\u001B[0;32m--> 531\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_init_message\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mcontext\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrecipient\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:334\u001B[0m, in \u001B[0;36mConversableAgent.send\u001B[0;34m(self, message, recipient, request_reply, silent)\u001B[0m\n\u001B[1;32m 332\u001B[0m valid \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_append_oai_message(message, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124massistant\u001B[39m\u001B[38;5;124m\"\u001B[39m, recipient)\n\u001B[1;32m 333\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m valid:\n\u001B[0;32m--> 334\u001B[0m \u001B[43mrecipient\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreceive\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessage\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrequest_reply\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 335\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 336\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 337\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mMessage can\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mt be converted into a valid ChatCompletion message. Either content or function_call must be provided.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 338\u001B[0m )\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:462\u001B[0m, in \u001B[0;36mConversableAgent.receive\u001B[0;34m(self, message, sender, request_reply, silent)\u001B[0m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mreply_at_receive[sender] \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m:\n\u001B[1;32m 461\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m\n\u001B[0;32m--> 462\u001B[0m reply \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_reply\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mchat_messages\u001B[49m\u001B[43m[\u001B[49m\u001B[43msender\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 463\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 464\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msend(reply, sender, silent\u001B[38;5;241m=\u001B[39msilent)\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:781\u001B[0m, in \u001B[0;36mConversableAgent.generate_reply\u001B[0;34m(self, messages, sender, exclude)\u001B[0m\n\u001B[1;32m 779\u001B[0m \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m 780\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_match_trigger(reply_func_tuple[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtrigger\u001B[39m\u001B[38;5;124m\"\u001B[39m], sender):\n\u001B[0;32m--> 781\u001B[0m final, reply \u001B[38;5;241m=\u001B[39m \u001B[43mreply_func\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreply_func_tuple\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mconfig\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 782\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final:\n\u001B[1;32m 783\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m reply\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/groupchat.py:162\u001B[0m, in \u001B[0;36mGroupChatManager.run_chat\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 159\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[1;32m 160\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 161\u001B[0m \u001B[38;5;66;03m# select the next speaker\u001B[39;00m\n\u001B[0;32m--> 162\u001B[0m speaker \u001B[38;5;241m=\u001B[39m \u001B[43mgroupchat\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mselect_speaker\u001B[49m\u001B[43m(\u001B[49m\u001B[43mspeaker\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;66;03m# let the speaker speak\u001B[39;00m\n\u001B[1;32m 164\u001B[0m reply \u001B[38;5;241m=\u001B[39m speaker\u001B[38;5;241m.\u001B[39mgenerate_reply(sender\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m)\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/groupchat.py:91\u001B[0m, in \u001B[0;36mGroupChat.select_speaker\u001B[0;34m(self, last_speaker, selector)\u001B[0m\n\u001B[1;32m 87\u001B[0m logger\u001B[38;5;241m.\u001B[39mwarning(\n\u001B[1;32m 88\u001B[0m \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mGroupChat is underpopulated with \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mn_agents\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m agents. Direct communication would be more efficient.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 89\u001B[0m )\n\u001B[1;32m 90\u001B[0m selector\u001B[38;5;241m.\u001B[39mupdate_system_message(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mselect_speaker_msg(agents))\n\u001B[0;32m---> 91\u001B[0m final, name \u001B[38;5;241m=\u001B[39m \u001B[43mselector\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_oai_reply\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 92\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmessages\u001B[49m\n\u001B[1;32m 93\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43m[\u001B[49m\n\u001B[1;32m 94\u001B[0m \u001B[43m \u001B[49m\u001B[43m{\u001B[49m\n\u001B[1;32m 95\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mrole\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43msystem\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 96\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontent\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43mf\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mRead the above conversation. Then select the next role from \u001B[39;49m\u001B[38;5;132;43;01m{\u001B[39;49;00m\u001B[43m[\u001B[49m\u001B[43magent\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mname\u001B[49m\u001B[38;5;250;43m \u001B[39;49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[38;5;250;43m \u001B[39;49m\u001B[43magent\u001B[49m\u001B[38;5;250;43m \u001B[39;49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[38;5;250;43m \u001B[39;49m\u001B[43magents\u001B[49m\u001B[43m]\u001B[49m\u001B[38;5;132;43;01m}\u001B[39;49;00m\u001B[38;5;124;43m to play. Only return the role.\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 97\u001B[0m \u001B[43m \u001B[49m\u001B[43m}\u001B[49m\n\u001B[1;32m 98\u001B[0m \u001B[43m \u001B[49m\u001B[43m]\u001B[49m\n\u001B[1;32m 99\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 100\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m final:\n\u001B[1;32m 101\u001B[0m \u001B[38;5;66;03m# i = self._random.randint(0, len(self._agent_names) - 1) # randomly pick an id\u001B[39;00m\n\u001B[1;32m 102\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mnext_agent(last_speaker, agents)\n", + "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:606\u001B[0m, in \u001B[0;36mConversableAgent.generate_oai_reply\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 603\u001B[0m messages \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_oai_messages[sender]\n\u001B[1;32m 605\u001B[0m \u001B[38;5;66;03m# TODO: #1143 handle token limit exceeded error\u001B[39;00m\n\u001B[0;32m--> 606\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43moai\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mChatCompletion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 607\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m-\u001B[39;49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpop\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontext\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_oai_system_message\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mllm_config\u001B[49m\n\u001B[1;32m 608\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 609\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m, oai\u001B[38;5;241m.\u001B[39mChatCompletion\u001B[38;5;241m.\u001B[39mextract_text_or_function_call(response)[\u001B[38;5;241m0\u001B[39m]\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:803\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 801\u001B[0m base_config[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmax_retry_period\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 802\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 803\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 804\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 805\u001B[0m \u001B[43m \u001B[49m\u001B[43muse_cache\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 806\u001B[0m \u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m<\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mlast\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01mor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 807\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mbase_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 808\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 809\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m response \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m 810\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m response\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:834\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 832\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m diskcache\u001B[38;5;241m.\u001B[39mCache(\u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mcache_path) \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39m_cache:\n\u001B[1;32m 833\u001B[0m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mset_cache(seed)\n\u001B[0;32m--> 834\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m)\u001B[49m\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:224\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 222\u001B[0m response \u001B[38;5;241m=\u001B[39m openai_completion\u001B[38;5;241m.\u001B[39mcreate(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mconfig)\n\u001B[1;32m 223\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 224\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mopenai_completion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 225\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\n\u001B[1;32m 226\u001B[0m ServiceUnavailableError,\n\u001B[1;32m 227\u001B[0m APIConnectionError,\n\u001B[1;32m 228\u001B[0m ):\n\u001B[1;32m 229\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n\u001B[1;32m 230\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mretrying in \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mretry_wait_time\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m seconds...\u001B[39m\u001B[38;5;124m\"\u001B[39m, exc_info\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py:25\u001B[0m, in \u001B[0;36mChatCompletion.create\u001B[0;34m(cls, *args, **kwargs)\u001B[0m\n\u001B[1;32m 23\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 25\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 26\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m TryAgain \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m timeout \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m>\u001B[39m start \u001B[38;5;241m+\u001B[39m timeout:\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001B[0m, in \u001B[0;36mEngineAPIResource.create\u001B[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001B[0m\n\u001B[1;32m 127\u001B[0m \u001B[38;5;129m@classmethod\u001B[39m\n\u001B[1;32m 128\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mcreate\u001B[39m(\n\u001B[1;32m 129\u001B[0m \u001B[38;5;28mcls\u001B[39m,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 136\u001B[0m \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams,\n\u001B[1;32m 137\u001B[0m ):\n\u001B[1;32m 138\u001B[0m (\n\u001B[1;32m 139\u001B[0m deployment_id,\n\u001B[1;32m 140\u001B[0m engine,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 150\u001B[0m api_key, api_base, api_type, api_version, organization, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams\n\u001B[1;32m 151\u001B[0m )\n\u001B[0;32m--> 153\u001B[0m response, _, api_key \u001B[38;5;241m=\u001B[39m \u001B[43mrequestor\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 154\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mpost\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 155\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 156\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 157\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 158\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 159\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 160\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 161\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream:\n\u001B[1;32m 164\u001B[0m \u001B[38;5;66;03m# must be an iterator\u001B[39;00m\n\u001B[1;32m 165\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(response, OpenAIResponse)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:288\u001B[0m, in \u001B[0;36mAPIRequestor.request\u001B[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 277\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mrequest\u001B[39m(\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28mself\u001B[39m,\n\u001B[1;32m 279\u001B[0m method,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 286\u001B[0m request_timeout: Optional[Union[\u001B[38;5;28mfloat\u001B[39m, Tuple[\u001B[38;5;28mfloat\u001B[39m, \u001B[38;5;28mfloat\u001B[39m]]] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m 287\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001B[38;5;28mbool\u001B[39m, \u001B[38;5;28mstr\u001B[39m]:\n\u001B[0;32m--> 288\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest_raw\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 289\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlower\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 290\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 291\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 292\u001B[0m \u001B[43m \u001B[49m\u001B[43msupplied_headers\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 293\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 294\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 295\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 296\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 297\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 298\u001B[0m resp, got_stream \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_interpret_response(result, stream)\n\u001B[1;32m 299\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp, got_stream, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mapi_key\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:596\u001B[0m, in \u001B[0;36mAPIRequestor.request_raw\u001B[0;34m(self, method, url, params, supplied_headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 594\u001B[0m _thread_context\u001B[38;5;241m.\u001B[39msession_create_time \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[1;32m 595\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 596\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 597\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 598\u001B[0m \u001B[43m \u001B[49m\u001B[43mabs_url\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 599\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 600\u001B[0m \u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 601\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 602\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 603\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mif\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01melse\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mTIMEOUT_SECS\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 604\u001B[0m \u001B[43m \u001B[49m\u001B[43mproxies\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mproxies\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 605\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 606\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m requests\u001B[38;5;241m.\u001B[39mexceptions\u001B[38;5;241m.\u001B[39mTimeout \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 607\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m error\u001B[38;5;241m.\u001B[39mTimeout(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mRequest timed out: \u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(e)) \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01me\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:589\u001B[0m, in \u001B[0;36mSession.request\u001B[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001B[0m\n\u001B[1;32m 584\u001B[0m send_kwargs \u001B[38;5;241m=\u001B[39m {\n\u001B[1;32m 585\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimeout\u001B[39m\u001B[38;5;124m\"\u001B[39m: timeout,\n\u001B[1;32m 586\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mallow_redirects\u001B[39m\u001B[38;5;124m\"\u001B[39m: allow_redirects,\n\u001B[1;32m 587\u001B[0m }\n\u001B[1;32m 588\u001B[0m send_kwargs\u001B[38;5;241m.\u001B[39mupdate(settings)\n\u001B[0;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mprep\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43msend_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 591\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:703\u001B[0m, in \u001B[0;36mSession.send\u001B[0;34m(self, request, **kwargs)\u001B[0m\n\u001B[1;32m 700\u001B[0m start \u001B[38;5;241m=\u001B[39m preferred_clock()\n\u001B[1;32m 702\u001B[0m \u001B[38;5;66;03m# Send the request\u001B[39;00m\n\u001B[0;32m--> 703\u001B[0m r \u001B[38;5;241m=\u001B[39m \u001B[43madapter\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 705\u001B[0m \u001B[38;5;66;03m# Total elapsed time of the request (approximately)\u001B[39;00m\n\u001B[1;32m 706\u001B[0m elapsed \u001B[38;5;241m=\u001B[39m preferred_clock() \u001B[38;5;241m-\u001B[39m start\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/adapters.py:486\u001B[0m, in \u001B[0;36mHTTPAdapter.send\u001B[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001B[0m\n\u001B[1;32m 483\u001B[0m timeout \u001B[38;5;241m=\u001B[39m TimeoutSauce(connect\u001B[38;5;241m=\u001B[39mtimeout, read\u001B[38;5;241m=\u001B[39mtimeout)\n\u001B[1;32m 485\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 486\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43murlopen\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 487\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 488\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 489\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 490\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 491\u001B[0m \u001B[43m \u001B[49m\u001B[43mredirect\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 492\u001B[0m \u001B[43m \u001B[49m\u001B[43massert_same_host\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 493\u001B[0m \u001B[43m \u001B[49m\u001B[43mpreload_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 494\u001B[0m \u001B[43m \u001B[49m\u001B[43mdecode_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 495\u001B[0m \u001B[43m \u001B[49m\u001B[43mretries\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmax_retries\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 496\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 497\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 498\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 500\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (ProtocolError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[1;32m 501\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m(err, request\u001B[38;5;241m=\u001B[39mrequest)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:714\u001B[0m, in \u001B[0;36mHTTPConnectionPool.urlopen\u001B[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001B[0m\n\u001B[1;32m 711\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_proxy(conn)\n\u001B[1;32m 713\u001B[0m \u001B[38;5;66;03m# Make the request on the httplib connection object.\u001B[39;00m\n\u001B[0;32m--> 714\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_make_request\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 715\u001B[0m \u001B[43m \u001B[49m\u001B[43mconn\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 716\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 717\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 718\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout_obj\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 719\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 720\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 721\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 722\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 724\u001B[0m \u001B[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001B[39;00m\n\u001B[1;32m 725\u001B[0m \u001B[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001B[39;00m\n\u001B[1;32m 726\u001B[0m \u001B[38;5;66;03m# it will also try to release it and we'll have a double-release\u001B[39;00m\n\u001B[1;32m 727\u001B[0m \u001B[38;5;66;03m# mess.\u001B[39;00m\n\u001B[1;32m 728\u001B[0m response_conn \u001B[38;5;241m=\u001B[39m conn \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m release_conn \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:466\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[0;32m--> 466\u001B[0m \u001B[43msix\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mraise_from\u001B[49m\u001B[43m(\u001B[49m\u001B[43me\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m 467\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 468\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_raise_timeout(err\u001B[38;5;241m=\u001B[39me, url\u001B[38;5;241m=\u001B[39murl, timeout_value\u001B[38;5;241m=\u001B[39mread_timeout)\n", + "File \u001B[0;32m:3\u001B[0m, in \u001B[0;36mraise_from\u001B[0;34m(value, from_value)\u001B[0m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:461\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 458\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n\u001B[1;32m 459\u001B[0m \u001B[38;5;66;03m# Python 3\u001B[39;00m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgetresponse\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[1;32m 466\u001B[0m six\u001B[38;5;241m.\u001B[39mraise_from(e, \u001B[38;5;28;01mNone\u001B[39;00m)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:1375\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 1373\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 1374\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1375\u001B[0m \u001B[43mresponse\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbegin\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1376\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m:\n\u001B[1;32m 1377\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclose()\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:318\u001B[0m, in \u001B[0;36mHTTPResponse.begin\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 316\u001B[0m \u001B[38;5;66;03m# read until we get a non-100 response\u001B[39;00m\n\u001B[1;32m 317\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m--> 318\u001B[0m version, status, reason \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_status\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 319\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m status \u001B[38;5;241m!=\u001B[39m CONTINUE:\n\u001B[1;32m 320\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:279\u001B[0m, in \u001B[0;36mHTTPResponse._read_status\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_read_status\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m--> 279\u001B[0m line \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mstr\u001B[39m(\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreadline\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_MAXLINE\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m)\u001B[49m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124miso-8859-1\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 280\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(line) \u001B[38;5;241m>\u001B[39m _MAXLINE:\n\u001B[1;32m 281\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m LineTooLong(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstatus line\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/socket.py:705\u001B[0m, in \u001B[0;36mSocketIO.readinto\u001B[0;34m(self, b)\u001B[0m\n\u001B[1;32m 703\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 704\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 705\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sock\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrecv_into\u001B[49m\u001B[43m(\u001B[49m\u001B[43mb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 706\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m timeout:\n\u001B[1;32m 707\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_timeout_occurred \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/ssl.py:1274\u001B[0m, in \u001B[0;36mSSLSocket.recv_into\u001B[0;34m(self, buffer, nbytes, flags)\u001B[0m\n\u001B[1;32m 1270\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m flags \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m 1271\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 1272\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m\n\u001B[1;32m 1273\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m)\n\u001B[0;32m-> 1274\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnbytes\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbuffer\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1275\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 1276\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39mrecv_into(buffer, nbytes, flags)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/ssl.py:1130\u001B[0m, in \u001B[0;36mSSLSocket.read\u001B[0;34m(self, len, buffer)\u001B[0m\n\u001B[1;32m 1128\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 1129\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m buffer \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m-> 1130\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sslobj\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mlen\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbuffer\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1131\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 1132\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sslobj\u001B[38;5;241m.\u001B[39mread(\u001B[38;5;28mlen\u001B[39m)\n", + "\u001B[0;31mKeyboardInterrupt\u001B[0m: " ] } ], @@ -539,16 +455,14 @@ " config_path=config_path\n", ")\n", "administrator.build(default_llm_config)\n", - "administrator.start(\n", - " initiate_agent_name=\"user\"\n", - ")\n", + "administrator.start()\n", "administrator.clear_all()\n" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-10-31T06:19:27.389747500Z", - "start_time": "2023-10-31T06:18:28.729808500Z" + "end_time": "2023-11-03T13:52:10.449037900Z", + "start_time": "2023-11-03T13:52:04.890203400Z" } }, "id": "7d52e3d9a1bf91cb" From 8ac9715797335cc08818655df0658f1c4f16b30d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 3 Nov 2023 22:57:00 +0900 Subject: [PATCH 08/74] add autobuild --- autobuild/__init__.py | 0 autobuild/agent_creator.py | 269 +++++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 autobuild/__init__.py create mode 100644 autobuild/agent_creator.py diff --git a/autobuild/__init__.py b/autobuild/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/autobuild/agent_creator.py b/autobuild/agent_creator.py new file mode 100644 index 00000000000..b3f15635789 --- /dev/null +++ b/autobuild/agent_creator.py @@ -0,0 +1,269 @@ +import autogen +import time +import subprocess as sp +import socket +from typing import * + + +class AgentCreator: + """ + Descriptions + """ + host: str + task: str + config_path: str + open_ports: List[str] = [] + agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} + openai_server_name: str = 'openai' + endpoint_building_timeout: Optional[int] + agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} + max_tokens: int = 945 + + user_proxy: autogen.UserProxyAgent = None + group_chat_manager_config: dict = None + initiate_agent_name: str = 'user' + manager_system_message: str = 'Group chat manager.' + + CODING_PROMPT: str = '''Does the following task need programming + (i.e., access external API or tool by coding) to solve? + + TASK: {task} + + Answer only YES or NO. + ''' + + def __init__( + self, + task: str, + host: str = 'localhost', + config_path: str = 'OAI_CONFIG_LIST', + endpoint_building_timeout: Optional[int] = 180 + ): + """ + Args: + task: description of a task. + endpoint_building_timeout: timeout for building up an endpoint server. + config_path: path of the OpenAI api configs. + host: endpoint host. + """ + self.task = task + self.endpoint_building_timeout = endpoint_building_timeout + self.config_path = config_path + self.host = host + + print('Initializing usable port...') + for port in range(8000, 65535): + if self._is_port_open(host, port): + self.open_ports.append(str(port)) + print(f'{len(self.open_ports)} ports found.') + + @staticmethod + def _is_port_open(host, port): + """Check if a port is open.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(10) + s.bind((host, int(port))) + s.close() + return True + except OSError: + return False + + def set_task(self, task: str): + self.task = task + + def create_agent( + self, + agent_name: str, + model_name_or_hf_repo: str, + llm_config: dict, + system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, + world_size: Optional[int] = 1 + ) -> autogen.AssistantAgent: + """ + Descriptions + + Args: + agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...) + model_name_or_hf_repo: + llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). + system_message: system prompt use to format an agent's behavior. + world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs). + + Returns: + agent: a set-up agent. + """ + config_list = autogen.config_list_from_json( + self.config_path, + filter_dict={ + 'model': [model_name_or_hf_repo] + } + ) + if 'gpt-' in model_name_or_hf_repo: + server_id = self.openai_server_name + else: + model_name = model_name_or_hf_repo.split('/')[-1] + server_id = f'{model_name}_{self.host}' + if self.agent_procs.get(server_id, None) is None: + while True: + port = self.open_ports.pop() + if self._is_port_open(self.host, port): + break + + # Use vLLM to set up a server with OpenAI API support. + agent_proc = sp.Popen(['python', '-m', 'vllm.entrypoints.openai.api_server', + f'--host', f'{self.host}', + f'--port', f'{port}', + f'--model', f'{model_name_or_hf_repo}', + f'--tensor-parallel-size', f'{world_size}'], stdout=sp.PIPE, stderr=sp.STDOUT) + timeout_start = time.time() + + while True: + server_stdout = agent_proc.stdout.readline() + if server_stdout != b'': + print(server_stdout) + timeout_end = time.time() + if b"running" in server_stdout: + print( + f'Running {model_name_or_hf_repo} on http://{self.host}:{port} ' + f'with tensor parallel size {world_size}.') + break + elif b"address already in use" in server_stdout: + raise RuntimeError(f'{self.host}:{port} already in use. Fail to set up the endpoint for ' + f'{model_name_or_hf_repo} on {self.host}:{port}.') + elif timeout_end - timeout_start > self.endpoint_building_timeout: + raise RuntimeError(f'Timeout exceed. Fail to set up the endpoint for ' + f'{model_name_or_hf_repo} on {self.host}:{port}.') + self.agent_procs[server_id] = (agent_proc, port) + else: + port = self.agent_procs[server_id][1] + + config_list[0]['api_base'] = f'http://{self.host}:{port}/v1' + + current_config = llm_config.copy() + current_config.update({ + 'config_list': config_list, + 'model': model_name_or_hf_repo, + 'max_tokens': self.max_tokens + }) + agent = autogen.AssistantAgent(name=agent_name, + llm_config=current_config.copy(), + system_message=system_message) + self.agent_procs_assign[agent_name] = (agent, server_id) + return agent + + def clear_agent( + self, + agent_name: str = None, + recycle_endpoint: bool = True + ): + """ + Descriptions + + Args: + agent_name: the name of agent. + recycle_endpoint: trigger for recycle the endpoint server. If true, the endpoint will be recycled + when there is no agent depending on. + """ + _, server_id = self.agent_procs_assign[agent_name] + del self.agent_procs_assign[agent_name] + if recycle_endpoint: + if server_id == self.openai_server_name: + return + else: + for _, iter_sid in self.agent_procs_assign.values(): + if server_id == iter_sid: + return + self.agent_procs[server_id][0].terminate() + self.open_ports.append(server_id.split('_')[-1]) + + def clear_all(self): + """ + Clear all cached agents. + """ + for agent_name in [agent_name for agent_name in self.agent_procs_assign.keys()]: + self.clear_agent(agent_name) + + def build( + self, + default_llm_config: dict, + coding: bool = None + ): + # TODO: not completed. + config_list = autogen.config_list_from_json( + self.config_path, + filter_dict={ + 'model': ['gpt-4'] + } + ) + agent_configs = [('Coder_gpt-35', 'gpt-3.5-turbo'), ('Product_manager', 'gpt-3.5-turbo')] + + for agent_config in agent_configs: + self.create_agent(agent_config[0], agent_config[1], default_llm_config) + + if coding is None: + resp = autogen.ChatCompletion.create( + messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.task)}], + **config_list[0] + ).choices[0].message.content + coding = True if resp == 'YES' else False + + if coding is True: + self.user_proxy = autogen.UserProxyAgent( + name="User_proxy", + system_message="A human admin.", + code_execution_config={"last_n_messages": 2, "work_dir": "groupchat"}, + human_input_mode="TERMINATE" + ) + else: + self.initiate_agent_name = agent_configs[0][0] + + self.group_chat_manager_config = default_llm_config.copy() + self.group_chat_manager_config['config_list'] = config_list + self.manager_system_message = 'Group chat manager.' + + def start( + self, + max_round: Optional[int] = 12, + init_messages: Optional[List[dict]] = [] + ): + """ + Descriptions + + Args: + max_round: the maximum number of rounds. + init_messages: input messages before the task start. This can be the chat history from other group chat + or some preliminary of the task. + initiate_agent_name: the name of an agent use to initialize the group chat. + """ + agent_list = [agent for agent, _ in self.agent_procs_assign.values()] + if self.user_proxy is not None: + agent_list.append(self.user_proxy) + group_chat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round) + + manager = autogen.GroupChatManager(groupchat=group_chat, + llm_config=self.group_chat_manager_config, + system_message=self.manager_system_message) + + if self.initiate_agent_name == "user" and self.user_proxy is not None: + self.user_proxy.initiate_chat(manager, message=self.task) + else: + for agent in agent_list: + if self.initiate_agent_name == agent.name(): + agent.initiate_chat(manager, message=self.task) + + +if __name__ == '__main__': + config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' + default_llm_config = { + 'temperature': 0 + } + + administrator = AgentCreator( + task="Find a latest paper about gpt-4 on arxiv and find its potential applications in software.", + config_path=config_path + ) + administrator.build(default_llm_config) + administrator.start() + administrator.clear_all() + From 8941d33a575fb911a898c4c88a761925e30da506 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 4 Nov 2023 17:01:24 +0900 Subject: [PATCH 09/74] rename autobuild notebook --- notebook/{oai_create_agent.ipynb => agentchat_autobuild.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename notebook/{oai_create_agent.ipynb => agentchat_autobuild.ipynb} (100%) diff --git a/notebook/oai_create_agent.ipynb b/notebook/agentchat_autobuild.ipynb similarity index 100% rename from notebook/oai_create_agent.ipynb rename to notebook/agentchat_autobuild.ipynb From 3aa249554f09ff025e8d2cb47f615e6e49badc4d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Tue, 7 Nov 2023 17:27:13 +0900 Subject: [PATCH 10/74] Add step-by-step command. --- autobuild/__init__.py | 1 + notebook/agentchat_autobuild.ipynb | 319 ++++------------------------- 2 files changed, 46 insertions(+), 274 deletions(-) diff --git a/autobuild/__init__.py b/autobuild/__init__.py index e69de29bb2d..848d98e7a2f 100644 --- a/autobuild/__init__.py +++ b/autobuild/__init__.py @@ -0,0 +1 @@ +from .agent_creator import AgentCreator \ No newline at end of file diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index e7e6885e742..7ca26a55efa 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -3,7 +3,11 @@ { "cell_type": "markdown", "source": [ - "# Multiple Agent Chat" + "# Multiple Agent Chat\n", + "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", + "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", + "\n", + "In this notebook, we introduce a new class, `AgentCreator`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our building pipeline include `init()`, `build()`, and `start()`. In `build()`, we prompt a gpt-4 model to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. " ], "metadata": { "collapsed": false @@ -13,11 +17,11 @@ { "cell_type": "markdown", "source": [ - "### Requirement\n", + "## Requirement\n", "\n", "Auto Agent setup need `vllm>=0.2.0` and `fastchat>=0.2.30`, you can use the following command to install the latest version of [vLLM](https://github.com/vllm-project/vllm) and [FastChat](https://github.com/lm-sys/FastChat).\n", "```\n", - "pip install vllm, \"fastchat[model_worker]\"\n", + "pip install vllm \"fastchat[model_worker]\"\n", "```" ], "metadata": { @@ -37,296 +41,65 @@ }, "id": "e8e9ae50658be975" }, + { + "cell_type": "markdown", + "source": [ + "## Specify a task with detailed information\n", + "Describe your task in natural language and specify your target (e.g., find some paper from arxiv)." + ], + "metadata": { + "collapsed": false + }, + "id": "3081403ca3b9b184" + }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "outputs": [], "source": [ - "import autogen\n", - "import time\n", - "import socket\n", - "import subprocess as sp\n", - "from typing import *" + "task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" ], "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-03T13:25:24.206940Z", - "start_time": "2023-11-03T13:25:24.205018900Z" - } + "collapsed": false }, - "id": "8d06a99bd05026bb" + "id": "97221d43eb23768" + }, + { + "cell_type": "markdown", + "source": [ + "## Prepare configuration\n", + "Prepare a `config_path` for assistant agent to limit the choice of LLM you want to use in this task. This config can be a path of json file or a name of environment variable. A `default_llm_config` is also required for initialize the specific config of LLMs like seed, temperature, etc..." + ], + "metadata": { + "collapsed": false + }, + "id": "7d0e63ab3604bdb9" }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "outputs": [], "source": [ - "class AgentCreator:\n", - " \"\"\"\n", - " Descriptions\n", - " \"\"\"\n", - " host: str\n", - " task: str\n", - " config_path: str\n", - " open_ports: List[str] = []\n", - " agent_procs: Dict[str, Tuple[sp.Popen, str]] = {}\n", - " openai_server_name: str = 'openai'\n", - " endpoint_building_timeout: Optional[int]\n", - " agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {}\n", - " max_tokens: int = 945\n", - "\n", - " user_proxy: autogen.UserProxyAgent = None\n", - " group_chat_manager_config: dict = None\n", - " initiate_agent_name: str = 'user'\n", - " manager_system_message: str = 'Group chat manager.'\n", - "\n", - " CODING_PROMPT: str = '''Does the following task need programming \n", - " (i.e., access external API or tool by coding) to solve?\n", - "\n", - " TASK: {task}\n", - "\n", - " Answer only YES or NO.\n", - " '''\n", - "\n", - " def __init__(\n", - " self,\n", - " task: str,\n", - " host: str = 'localhost',\n", - " config_path: str = 'OAI_CONFIG_LIST',\n", - " endpoint_building_timeout: Optional[int] = 180\n", - " ):\n", - " \"\"\"\n", - " Args:\n", - " task: description of a task.\n", - " endpoint_building_timeout: timeout for building up an endpoint server.\n", - " config_path: path of the OpenAI api configs.\n", - " host: endpoint host.\n", - " \"\"\"\n", - " self.task = task\n", - " self.endpoint_building_timeout = endpoint_building_timeout\n", - " self.config_path = config_path\n", - " self.host = host\n", - "\n", - " print('Initializing usable port...')\n", - " for port in range(8000, 65535):\n", - " if self._is_port_open(host, port):\n", - " self.open_ports.append(str(port))\n", - " print(f'{len(self.open_ports)} ports found.')\n", - "\n", - " @staticmethod\n", - " def _is_port_open(host, port):\n", - " \"\"\"Check if a port is open.\"\"\"\n", - " try:\n", - " s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n", - " s.settimeout(10)\n", - " s.bind((host, int(port)))\n", - " s.close()\n", - " return True\n", - " except OSError:\n", - " return False\n", - "\n", - " def set_task(self, task: str):\n", - " self.task = task\n", - "\n", - " def create_agent(\n", - " self,\n", - " agent_name: str,\n", - " model_name_or_hf_repo: str,\n", - " llm_config: dict,\n", - " system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE,\n", - " world_size: Optional[int] = 1\n", - " ) -> autogen.AssistantAgent:\n", - " \"\"\"\n", - " Descriptions\n", - "\n", - " Args:\n", - " agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...)\n", - " model_name_or_hf_repo:\n", - " llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...).\n", - " system_message: system prompt use to format an agent's behavior.\n", - " world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs).\n", - "\n", - " Returns:\n", - " agent: a set-up agent.\n", - " \"\"\"\n", - " config_list = autogen.config_list_from_json(\n", - " self.config_path,\n", - " filter_dict={\n", - " 'model': [model_name_or_hf_repo]\n", - " }\n", - " )\n", - " if 'gpt-' in model_name_or_hf_repo:\n", - " server_id = self.openai_server_name\n", - " else:\n", - " model_name = model_name_or_hf_repo.split('/')[-1]\n", - " server_id = f'{model_name}_{self.host}'\n", - " if self.agent_procs.get(server_id, None) is None:\n", - " while True:\n", - " port = self.open_ports.pop()\n", - " if self._is_port_open(self.host, port):\n", - " break\n", - "\n", - " # Use vLLM to set up a server with OpenAI API support.\n", - " agent_proc = sp.Popen(['python', '-m', 'vllm.entrypoints.openai.api_server',\n", - " f'--host', f'{self.host}',\n", - " f'--port', f'{port}',\n", - " f'--model', f'{model_name_or_hf_repo}',\n", - " f'--tensor-parallel-size', f'{world_size}'], stdout=sp.PIPE, stderr=sp.STDOUT)\n", - " timeout_start = time.time()\n", - "\n", - " while True:\n", - " server_stdout = agent_proc.stdout.readline()\n", - " if server_stdout != b'':\n", - " print(server_stdout)\n", - " timeout_end = time.time()\n", - " if b\"running\" in server_stdout:\n", - " print(\n", - " f'Running {model_name_or_hf_repo} on http://{self.host}:{port} '\n", - " f'with tensor parallel size {world_size}.')\n", - " break\n", - " elif b\"address already in use\" in server_stdout:\n", - " raise RuntimeError(f'{self.host}:{port} already in use. Fail to set up the endpoint for '\n", - " f'{model_name_or_hf_repo} on {self.host}:{port}.')\n", - " elif timeout_end - timeout_start > self.endpoint_building_timeout:\n", - " raise RuntimeError(f'Timeout exceed. Fail to set up the endpoint for '\n", - " f'{model_name_or_hf_repo} on {self.host}:{port}.')\n", - " self.agent_procs[server_id] = (agent_proc, port)\n", - " else:\n", - " port = self.agent_procs[server_id][1]\n", - "\n", - " config_list[0]['api_base'] = f'http://{self.host}:{port}/v1'\n", - "\n", - " current_config = llm_config.copy()\n", - " current_config.update({\n", - " 'config_list': config_list,\n", - " 'model': model_name_or_hf_repo,\n", - " 'max_tokens': self.max_tokens\n", - " })\n", - " agent = autogen.AssistantAgent(name=agent_name,\n", - " llm_config=current_config.copy(),\n", - " system_message=system_message)\n", - " self.agent_procs_assign[agent_name] = (agent, server_id)\n", - " return agent\n", - "\n", - " def clear_agent(\n", - " self,\n", - " agent_name: str = None,\n", - " recycle_endpoint: bool = True\n", - " ):\n", - " \"\"\"\n", - " Descriptions\n", - "\n", - " Args:\n", - " agent_name: the name of agent.\n", - " recycle_endpoint: trigger for recycle the endpoint server. If true, the endpoint will be recycled\n", - " when there is no agent depending on.\n", - " \"\"\"\n", - " _, server_id = self.agent_procs_assign[agent_name]\n", - " del self.agent_procs_assign[agent_name]\n", - " if recycle_endpoint:\n", - " if server_id == self.openai_server_name:\n", - " return\n", - " else:\n", - " for _, iter_sid in self.agent_procs_assign.values():\n", - " if server_id == iter_sid:\n", - " return\n", - " self.agent_procs[server_id][0].terminate()\n", - " self.open_ports.append(server_id.split('_')[-1])\n", - "\n", - " def clear_all(self):\n", - " \"\"\"\n", - " Clear all cached agents.\n", - " \"\"\"\n", - " for agent_name in [agent_name for agent_name in self.agent_procs_assign.keys()]:\n", - " self.clear_agent(agent_name)\n", - "\n", - " def build(\n", - " self,\n", - " default_llm_config: dict,\n", - " coding: bool = None\n", - " ):\n", - " # TODO: not completed.\n", - " config_list = autogen.config_list_from_json(\n", - " self.config_path,\n", - " filter_dict={\n", - " 'model': ['gpt-4']\n", - " }\n", - " )\n", - " agent_configs = [('Coder_gpt-35', 'gpt-3.5-turbo'), ('Product_manager', 'gpt-3.5-turbo')]\n", - " \n", - " for agent_config in agent_configs:\n", - " self.create_agent(agent_config[0], agent_config[1], default_llm_config)\n", - " \n", - " if coding is None:\n", - " resp = autogen.ChatCompletion.create(\n", - " messages=[{\"role\": \"user\", \"content\": self.CODING_PROMPT.format(task=self.task)}],\n", - " **config_list[0]\n", - " ).choices[0].message.content\n", - " coding = True if resp == 'YES' else False\n", - "\n", - " if coding is True:\n", - " self.user_proxy = autogen.UserProxyAgent(\n", - " name=\"User_proxy\",\n", - " system_message=\"A human admin.\",\n", - " code_execution_config={\"last_n_messages\": 2, \"work_dir\": \"groupchat\"},\n", - " human_input_mode=\"TERMINATE\"\n", - " )\n", - " else:\n", - " self.initiate_agent_name = agent_configs[0][0]\n", - " \n", - " self.group_chat_manager_config = default_llm_config.copy()\n", - " self.group_chat_manager_config['config_list'] = config_list\n", - " self.manager_system_message = 'Group chat manager.'\n", - "\n", - " def start(\n", - " self,\n", - " max_round: Optional[int] = 12,\n", - " init_messages: Optional[List[dict]] = []\n", - " ):\n", - " \"\"\"\n", - " Descriptions\n", - "\n", - " Args:\n", - " max_round: the maximum number of rounds.\n", - " init_messages: input messages before the task start. This can be the chat history from other group chat\n", - " or some preliminary of the task.\n", - " initiate_agent_name: the name of an agent use to initialize the group chat.\n", - " \"\"\"\n", - " agent_list = [agent for agent, _ in self.agent_procs_assign.values()]\n", - " if self.user_proxy is not None:\n", - " agent_list.append(self.user_proxy)\n", - " group_chat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round)\n", - "\n", - " manager = autogen.GroupChatManager(groupchat=group_chat,\n", - " llm_config=self.group_chat_manager_config,\n", - " system_message=self.manager_system_message)\n", - "\n", - " if self.initiate_agent_name == \"user\" and self.user_proxy is not None:\n", - " self.user_proxy.initiate_chat(manager, message=self.task)\n", - " else:\n", - " for agent in agent_list:\n", - " if self.initiate_agent_name == agent.name():\n", - " agent.initiate_chat(manager, message=self.task)" + "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", + "default_llm_config = {\n", + " 'temperature': 0\n", + "}" ], "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-03T13:51:43.383138700Z", - "start_time": "2023-11-03T13:51:43.367423200Z" - } + "collapsed": false }, - "id": "d729233c9524f875" + "id": "2505f029423b21ab" }, { "cell_type": "markdown", "source": [ - "# Test (GPT)" + "## Complete the task building\n", + "Then, we can use the `AgentCreator` to automatically complete the above task. First, initialize the AgentCreator with the predefined `task` and `config_path`. Then, call the `build` with the `default_llm_config`. GPT-4 will automatically create multiple agents and initialize a groupchat that use to complete this task. Finally, call the `start()` to start the task completion process with a groupchat. When the task begin, groupchat content between each agent will be shown in the console. This process will become interactive when the groupchat manager think that is necessary. Note that if this process involve some open-sourced LLMs (for example, LLaMA 2), an endpoint server will be automatically set up with a background process. You can call the `clear_all()` to wipe them out after the task completed." ], "metadata": { "collapsed": false }, - "id": "3d4b09d29b740bf7" + "id": "c2d6586c68fa425b" }, { "cell_type": "code", @@ -445,10 +218,8 @@ } ], "source": [ - "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", - "default_llm_config = {\n", - " 'temperature': 0\n", - "}\n", + "from autobuild import AgentCreator\n", + "\n", "\n", "administrator = AgentCreator(\n", " task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\",\n", From dbfe0c44a48721808d2a485c4c043f72160fe71d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Tue, 7 Nov 2023 17:29:36 +0900 Subject: [PATCH 11/74] modify name --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 7ca26a55efa..a4e24fe9e70 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -3,7 +3,7 @@ { "cell_type": "markdown", "source": [ - "# Multiple Agent Chat\n", + "# Auto Builder\n", "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", From 34684f1ca5f1641717bc01f6048ea84fb810f8e3 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Tue, 7 Nov 2023 17:31:34 +0900 Subject: [PATCH 12/74] fix bugs --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index a4e24fe9e70..ee022ab4133 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -222,7 +222,7 @@ "\n", "\n", "administrator = AgentCreator(\n", - " task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\",\n", + " task,\n", " config_path=config_path\n", ")\n", "administrator.build(default_llm_config)\n", From f3ba3f7eec3be3642744a654c898feda2d3ea8ed Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 8 Nov 2023 23:31:35 +0900 Subject: [PATCH 13/74] update for new openai api --- autobuild/agent_creator.py | 6 +- notebook/agentchat_autobuild.ipynb | 130 ++++++----------------------- 2 files changed, 28 insertions(+), 108 deletions(-) diff --git a/autobuild/agent_creator.py b/autobuild/agent_creator.py index b3f15635789..44eb9a95ac4 100644 --- a/autobuild/agent_creator.py +++ b/autobuild/agent_creator.py @@ -202,9 +202,9 @@ def build( self.create_agent(agent_config[0], agent_config[1], default_llm_config) if coding is None: - resp = autogen.ChatCompletion.create( - messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.task)}], - **config_list[0] + api = autogen.OpenAIWrapper(config_list=config_list) + resp = api.create( + messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.task)}] ).choices[0].message.content coding = True if resp == 'YES' else False diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index ee022ab4133..b01be8853d6 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -54,13 +54,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "outputs": [], "source": [ "task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" ], "metadata": { - "collapsed": false + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-08T14:25:51.075351900Z", + "start_time": "2023-11-08T14:25:51.069815900Z" + } }, "id": "97221d43eb23768" }, @@ -77,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "outputs": [], "source": [ "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", @@ -86,7 +90,11 @@ "}" ], "metadata": { - "collapsed": false + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-08T14:25:51.612271500Z", + "start_time": "2023-11-08T14:25:51.603397800Z" + } }, "id": "2505f029423b21ab" }, @@ -103,117 +111,29 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 3, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initializing usable port...\n", - "115036 ports found.\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", - "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the abstract or content of the paper to identify its potential applications in software.\n", - "\n", - "Here is the Python code to accomplish this task:\n", - "\n", - "```python\n", - "import requests\n", - "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query={search_query}&sortBy=submittedDate&sortOrder=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", - "\n", - "# Parse the XML response and extract the paper details\n", - "xml_data = response.text\n", - "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "paper_abstract = xml_data.split(\"\")[1].split(\"\")[0]\n", - "\n", - "# Print the paper details\n", - "print(\"Latest Paper on GPT-4:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Abstract:\", paper_abstract)\n", - "```\n", - "\n", - "Please note that the above code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", - "\n", - "After executing the code, it will print the title and abstract of the latest paper on GPT-4. You can then analyze the abstract to identify potential applications in software.\n", - "\n", - "Please let me know if you need any further assistance.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "exitcode: 1 (execution failed)\n", - "Code output: \n", - "Traceback (most recent call last):\n", - " File \"\", line 10, in \n", - " paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "IndexError: list index out of range\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n" + "57505 ports found.\n" ] }, { - "ename": "KeyboardInterrupt", - "evalue": "", + "ename": "ModuleNotFoundError", + "evalue": "No module named 'openai.openai_object'", "output_type": "error", "traceback": [ "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[34], line 11\u001B[0m\n\u001B[1;32m 6\u001B[0m administrator \u001B[38;5;241m=\u001B[39m AgentCreator(\n\u001B[1;32m 7\u001B[0m task\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m 8\u001B[0m config_path\u001B[38;5;241m=\u001B[39mconfig_path\n\u001B[1;32m 9\u001B[0m )\n\u001B[1;32m 10\u001B[0m administrator\u001B[38;5;241m.\u001B[39mbuild(default_llm_config)\n\u001B[0;32m---> 11\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstart\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 12\u001B[0m administrator\u001B[38;5;241m.\u001B[39mclear_all()\n", - "Cell \u001B[0;32mIn[32], line 242\u001B[0m, in \u001B[0;36mAgentCreator.start\u001B[0;34m(self, max_round, init_messages)\u001B[0m\n\u001B[1;32m 237\u001B[0m manager \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mGroupChatManager(groupchat\u001B[38;5;241m=\u001B[39mgroup_chat,\n\u001B[1;32m 238\u001B[0m llm_config\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mgroup_chat_manager_config,\n\u001B[1;32m 239\u001B[0m system_message\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmanager_system_message)\n\u001B[1;32m 241\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39minitiate_agent_name \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124muser\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39muser_proxy \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m--> 242\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43muser_proxy\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minitiate_chat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmanager\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 243\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 244\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m agent \u001B[38;5;129;01min\u001B[39;00m agent_list:\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:531\u001B[0m, in \u001B[0;36mConversableAgent.initiate_chat\u001B[0;34m(self, recipient, clear_history, silent, **context)\u001B[0m\n\u001B[1;32m 517\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"Initiate a chat with the recipient agent.\u001B[39;00m\n\u001B[1;32m 518\u001B[0m \n\u001B[1;32m 519\u001B[0m \u001B[38;5;124;03mReset the consecutive auto reply counter.\u001B[39;00m\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 528\u001B[0m \u001B[38;5;124;03m \"message\" needs to be provided if the `generate_init_message` method is not overridden.\u001B[39;00m\n\u001B[1;32m 529\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m 530\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_chat(recipient, clear_history)\n\u001B[0;32m--> 531\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_init_message\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mcontext\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrecipient\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:334\u001B[0m, in \u001B[0;36mConversableAgent.send\u001B[0;34m(self, message, recipient, request_reply, silent)\u001B[0m\n\u001B[1;32m 332\u001B[0m valid \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_append_oai_message(message, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124massistant\u001B[39m\u001B[38;5;124m\"\u001B[39m, recipient)\n\u001B[1;32m 333\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m valid:\n\u001B[0;32m--> 334\u001B[0m \u001B[43mrecipient\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreceive\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessage\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrequest_reply\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msilent\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 335\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 336\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 337\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mMessage can\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mt be converted into a valid ChatCompletion message. Either content or function_call must be provided.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 338\u001B[0m )\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:462\u001B[0m, in \u001B[0;36mConversableAgent.receive\u001B[0;34m(self, message, sender, request_reply, silent)\u001B[0m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m request_reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mreply_at_receive[sender] \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m:\n\u001B[1;32m 461\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m\n\u001B[0;32m--> 462\u001B[0m reply \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_reply\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mchat_messages\u001B[49m\u001B[43m[\u001B[49m\u001B[43msender\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 463\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m reply \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 464\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msend(reply, sender, silent\u001B[38;5;241m=\u001B[39msilent)\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:781\u001B[0m, in \u001B[0;36mConversableAgent.generate_reply\u001B[0;34m(self, messages, sender, exclude)\u001B[0m\n\u001B[1;32m 779\u001B[0m \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m 780\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_match_trigger(reply_func_tuple[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtrigger\u001B[39m\u001B[38;5;124m\"\u001B[39m], sender):\n\u001B[0;32m--> 781\u001B[0m final, reply \u001B[38;5;241m=\u001B[39m \u001B[43mreply_func\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msender\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msender\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreply_func_tuple\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mconfig\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 782\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final:\n\u001B[1;32m 783\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m reply\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/groupchat.py:162\u001B[0m, in \u001B[0;36mGroupChatManager.run_chat\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 159\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[1;32m 160\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 161\u001B[0m \u001B[38;5;66;03m# select the next speaker\u001B[39;00m\n\u001B[0;32m--> 162\u001B[0m speaker \u001B[38;5;241m=\u001B[39m \u001B[43mgroupchat\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mselect_speaker\u001B[49m\u001B[43m(\u001B[49m\u001B[43mspeaker\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;66;03m# let the speaker speak\u001B[39;00m\n\u001B[1;32m 164\u001B[0m reply \u001B[38;5;241m=\u001B[39m speaker\u001B[38;5;241m.\u001B[39mgenerate_reply(sender\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m)\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/groupchat.py:91\u001B[0m, in \u001B[0;36mGroupChat.select_speaker\u001B[0;34m(self, last_speaker, selector)\u001B[0m\n\u001B[1;32m 87\u001B[0m logger\u001B[38;5;241m.\u001B[39mwarning(\n\u001B[1;32m 88\u001B[0m \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mGroupChat is underpopulated with \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mn_agents\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m agents. Direct communication would be more efficient.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 89\u001B[0m )\n\u001B[1;32m 90\u001B[0m selector\u001B[38;5;241m.\u001B[39mupdate_system_message(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mselect_speaker_msg(agents))\n\u001B[0;32m---> 91\u001B[0m final, name \u001B[38;5;241m=\u001B[39m \u001B[43mselector\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgenerate_oai_reply\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 92\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmessages\u001B[49m\n\u001B[1;32m 93\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43m[\u001B[49m\n\u001B[1;32m 94\u001B[0m \u001B[43m \u001B[49m\u001B[43m{\u001B[49m\n\u001B[1;32m 95\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mrole\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43msystem\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 96\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontent\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43mf\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mRead the above conversation. Then select the next role from \u001B[39;49m\u001B[38;5;132;43;01m{\u001B[39;49;00m\u001B[43m[\u001B[49m\u001B[43magent\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mname\u001B[49m\u001B[38;5;250;43m \u001B[39;49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[38;5;250;43m \u001B[39;49m\u001B[43magent\u001B[49m\u001B[38;5;250;43m \u001B[39;49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[38;5;250;43m \u001B[39;49m\u001B[43magents\u001B[49m\u001B[43m]\u001B[49m\u001B[38;5;132;43;01m}\u001B[39;49;00m\u001B[38;5;124;43m to play. Only return the role.\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 97\u001B[0m \u001B[43m \u001B[49m\u001B[43m}\u001B[49m\n\u001B[1;32m 98\u001B[0m \u001B[43m \u001B[49m\u001B[43m]\u001B[49m\n\u001B[1;32m 99\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 100\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m final:\n\u001B[1;32m 101\u001B[0m \u001B[38;5;66;03m# i = self._random.randint(0, len(self._agent_names) - 1) # randomly pick an id\u001B[39;00m\n\u001B[1;32m 102\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mnext_agent(last_speaker, agents)\n", - "File \u001B[0;32m~/LLM/autogen/autogen/agentchat/conversable_agent.py:606\u001B[0m, in \u001B[0;36mConversableAgent.generate_oai_reply\u001B[0;34m(self, messages, sender, config)\u001B[0m\n\u001B[1;32m 603\u001B[0m messages \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_oai_messages[sender]\n\u001B[1;32m 605\u001B[0m \u001B[38;5;66;03m# TODO: #1143 handle token limit exceeded error\u001B[39;00m\n\u001B[0;32m--> 606\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43moai\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mChatCompletion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 607\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmessages\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m-\u001B[39;49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpop\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontext\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_oai_system_message\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mllm_config\u001B[49m\n\u001B[1;32m 608\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 609\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m, oai\u001B[38;5;241m.\u001B[39mChatCompletion\u001B[38;5;241m.\u001B[39mextract_text_or_function_call(response)[\u001B[38;5;241m0\u001B[39m]\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:803\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 801\u001B[0m base_config[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmax_retry_period\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m 802\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 803\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 804\u001B[0m \u001B[43m \u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 805\u001B[0m \u001B[43m \u001B[49m\u001B[43muse_cache\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 806\u001B[0m \u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m<\u001B[39;49m\u001B[43m \u001B[49m\u001B[43mlast\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01mor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 807\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mbase_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 808\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 809\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m response \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m:\n\u001B[1;32m 810\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m response\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:834\u001B[0m, in \u001B[0;36mCompletion.create\u001B[0;34m(cls, context, use_cache, config_list, filter_func, raise_on_ratelimit_or_timeout, allow_format_str_template, **config)\u001B[0m\n\u001B[1;32m 832\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m diskcache\u001B[38;5;241m.\u001B[39mCache(\u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mcache_path) \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39m_cache:\n\u001B[1;32m 833\u001B[0m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39mset_cache(seed)\n\u001B[0;32m--> 834\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_get_response\u001B[49m\u001B[43m(\u001B[49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mraise_on_ratelimit_or_timeout\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/completion.py:224\u001B[0m, in \u001B[0;36mCompletion._get_response\u001B[0;34m(cls, config, raise_on_ratelimit_or_timeout, use_cache)\u001B[0m\n\u001B[1;32m 222\u001B[0m response \u001B[38;5;241m=\u001B[39m openai_completion\u001B[38;5;241m.\u001B[39mcreate(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mconfig)\n\u001B[1;32m 223\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 224\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mopenai_completion\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 225\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\n\u001B[1;32m 226\u001B[0m ServiceUnavailableError,\n\u001B[1;32m 227\u001B[0m APIConnectionError,\n\u001B[1;32m 228\u001B[0m ):\n\u001B[1;32m 229\u001B[0m \u001B[38;5;66;03m# transient error\u001B[39;00m\n\u001B[1;32m 230\u001B[0m logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mretrying in \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mretry_wait_time\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m seconds...\u001B[39m\u001B[38;5;124m\"\u001B[39m, exc_info\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/chat_completion.py:25\u001B[0m, in \u001B[0;36mChatCompletion.create\u001B[0;34m(cls, *args, **kwargs)\u001B[0m\n\u001B[1;32m 23\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 25\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 26\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m TryAgain \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 27\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m timeout \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m>\u001B[39m start \u001B[38;5;241m+\u001B[39m timeout:\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001B[0m, in \u001B[0;36mEngineAPIResource.create\u001B[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001B[0m\n\u001B[1;32m 127\u001B[0m \u001B[38;5;129m@classmethod\u001B[39m\n\u001B[1;32m 128\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mcreate\u001B[39m(\n\u001B[1;32m 129\u001B[0m \u001B[38;5;28mcls\u001B[39m,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 136\u001B[0m \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams,\n\u001B[1;32m 137\u001B[0m ):\n\u001B[1;32m 138\u001B[0m (\n\u001B[1;32m 139\u001B[0m deployment_id,\n\u001B[1;32m 140\u001B[0m engine,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 150\u001B[0m api_key, api_base, api_type, api_version, organization, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mparams\n\u001B[1;32m 151\u001B[0m )\n\u001B[0;32m--> 153\u001B[0m response, _, api_key \u001B[38;5;241m=\u001B[39m \u001B[43mrequestor\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 154\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mpost\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m 155\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 156\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 157\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 158\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 159\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 160\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 161\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 163\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m stream:\n\u001B[1;32m 164\u001B[0m \u001B[38;5;66;03m# must be an iterator\u001B[39;00m\n\u001B[1;32m 165\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(response, OpenAIResponse)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:288\u001B[0m, in \u001B[0;36mAPIRequestor.request\u001B[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 277\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mrequest\u001B[39m(\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28mself\u001B[39m,\n\u001B[1;32m 279\u001B[0m method,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 286\u001B[0m request_timeout: Optional[Union[\u001B[38;5;28mfloat\u001B[39m, Tuple[\u001B[38;5;28mfloat\u001B[39m, \u001B[38;5;28mfloat\u001B[39m]]] \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m 287\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001B[38;5;28mbool\u001B[39m, \u001B[38;5;28mstr\u001B[39m]:\n\u001B[0;32m--> 288\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest_raw\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 289\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mlower\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 290\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 291\u001B[0m \u001B[43m \u001B[49m\u001B[43mparams\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mparams\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 292\u001B[0m \u001B[43m \u001B[49m\u001B[43msupplied_headers\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 293\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 294\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 295\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_id\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 296\u001B[0m \u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 297\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 298\u001B[0m resp, got_stream \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_interpret_response(result, stream)\n\u001B[1;32m 299\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp, got_stream, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mapi_key\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/openai/api_requestor.py:596\u001B[0m, in \u001B[0;36mAPIRequestor.request_raw\u001B[0;34m(self, method, url, params, supplied_headers, files, stream, request_id, request_timeout)\u001B[0m\n\u001B[1;32m 594\u001B[0m _thread_context\u001B[38;5;241m.\u001B[39msession_create_time \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime()\n\u001B[1;32m 595\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 596\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 597\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 598\u001B[0m \u001B[43m \u001B[49m\u001B[43mabs_url\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 599\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 600\u001B[0m \u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 601\u001B[0m \u001B[43m \u001B[49m\u001B[43mfiles\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfiles\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 602\u001B[0m \u001B[43m \u001B[49m\u001B[43mstream\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstream\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 603\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mif\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mrequest_timeout\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01melse\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mTIMEOUT_SECS\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 604\u001B[0m \u001B[43m \u001B[49m\u001B[43mproxies\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m_thread_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mproxies\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 605\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 606\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m requests\u001B[38;5;241m.\u001B[39mexceptions\u001B[38;5;241m.\u001B[39mTimeout \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 607\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m error\u001B[38;5;241m.\u001B[39mTimeout(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mRequest timed out: \u001B[39m\u001B[38;5;132;01m{}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mformat(e)) \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01me\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:589\u001B[0m, in \u001B[0;36mSession.request\u001B[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001B[0m\n\u001B[1;32m 584\u001B[0m send_kwargs \u001B[38;5;241m=\u001B[39m {\n\u001B[1;32m 585\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimeout\u001B[39m\u001B[38;5;124m\"\u001B[39m: timeout,\n\u001B[1;32m 586\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mallow_redirects\u001B[39m\u001B[38;5;124m\"\u001B[39m: allow_redirects,\n\u001B[1;32m 587\u001B[0m }\n\u001B[1;32m 588\u001B[0m send_kwargs\u001B[38;5;241m.\u001B[39mupdate(settings)\n\u001B[0;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mprep\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43msend_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 591\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/sessions.py:703\u001B[0m, in \u001B[0;36mSession.send\u001B[0;34m(self, request, **kwargs)\u001B[0m\n\u001B[1;32m 700\u001B[0m start \u001B[38;5;241m=\u001B[39m preferred_clock()\n\u001B[1;32m 702\u001B[0m \u001B[38;5;66;03m# Send the request\u001B[39;00m\n\u001B[0;32m--> 703\u001B[0m r \u001B[38;5;241m=\u001B[39m \u001B[43madapter\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequest\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 705\u001B[0m \u001B[38;5;66;03m# Total elapsed time of the request (approximately)\u001B[39;00m\n\u001B[1;32m 706\u001B[0m elapsed \u001B[38;5;241m=\u001B[39m preferred_clock() \u001B[38;5;241m-\u001B[39m start\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/requests/adapters.py:486\u001B[0m, in \u001B[0;36mHTTPAdapter.send\u001B[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001B[0m\n\u001B[1;32m 483\u001B[0m timeout \u001B[38;5;241m=\u001B[39m TimeoutSauce(connect\u001B[38;5;241m=\u001B[39mtimeout, read\u001B[38;5;241m=\u001B[39mtimeout)\n\u001B[1;32m 485\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 486\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43murlopen\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 487\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 488\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 489\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 490\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 491\u001B[0m \u001B[43m \u001B[49m\u001B[43mredirect\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 492\u001B[0m \u001B[43m \u001B[49m\u001B[43massert_same_host\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 493\u001B[0m \u001B[43m \u001B[49m\u001B[43mpreload_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 494\u001B[0m \u001B[43m \u001B[49m\u001B[43mdecode_content\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m 495\u001B[0m \u001B[43m \u001B[49m\u001B[43mretries\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmax_retries\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 496\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 497\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 498\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 500\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (ProtocolError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[1;32m 501\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m(err, request\u001B[38;5;241m=\u001B[39mrequest)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:714\u001B[0m, in \u001B[0;36mHTTPConnectionPool.urlopen\u001B[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001B[0m\n\u001B[1;32m 711\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_prepare_proxy(conn)\n\u001B[1;32m 713\u001B[0m \u001B[38;5;66;03m# Make the request on the httplib connection object.\u001B[39;00m\n\u001B[0;32m--> 714\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_make_request\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 715\u001B[0m \u001B[43m \u001B[49m\u001B[43mconn\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 716\u001B[0m \u001B[43m \u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 717\u001B[0m \u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 718\u001B[0m \u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtimeout_obj\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 719\u001B[0m \u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 720\u001B[0m \u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 721\u001B[0m \u001B[43m \u001B[49m\u001B[43mchunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunked\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 722\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 724\u001B[0m \u001B[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001B[39;00m\n\u001B[1;32m 725\u001B[0m \u001B[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001B[39;00m\n\u001B[1;32m 726\u001B[0m \u001B[38;5;66;03m# it will also try to release it and we'll have a double-release\u001B[39;00m\n\u001B[1;32m 727\u001B[0m \u001B[38;5;66;03m# mess.\u001B[39;00m\n\u001B[1;32m 728\u001B[0m response_conn \u001B[38;5;241m=\u001B[39m conn \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m release_conn \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:466\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[0;32m--> 466\u001B[0m \u001B[43msix\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mraise_from\u001B[49m\u001B[43m(\u001B[49m\u001B[43me\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m 467\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 468\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_raise_timeout(err\u001B[38;5;241m=\u001B[39me, url\u001B[38;5;241m=\u001B[39murl, timeout_value\u001B[38;5;241m=\u001B[39mread_timeout)\n", - "File \u001B[0;32m:3\u001B[0m, in \u001B[0;36mraise_from\u001B[0;34m(value, from_value)\u001B[0m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/urllib3/connectionpool.py:461\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001B[0m\n\u001B[1;32m 458\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n\u001B[1;32m 459\u001B[0m \u001B[38;5;66;03m# Python 3\u001B[39;00m\n\u001B[1;32m 460\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 461\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[43mconn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgetresponse\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 462\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mBaseException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m 463\u001B[0m \u001B[38;5;66;03m# Remove the TypeError from the exception chain in\u001B[39;00m\n\u001B[1;32m 464\u001B[0m \u001B[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001B[39;00m\n\u001B[1;32m 465\u001B[0m \u001B[38;5;66;03m# Otherwise it looks like a bug in the code.\u001B[39;00m\n\u001B[1;32m 466\u001B[0m six\u001B[38;5;241m.\u001B[39mraise_from(e, \u001B[38;5;28;01mNone\u001B[39;00m)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:1375\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 1373\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 1374\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1375\u001B[0m \u001B[43mresponse\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbegin\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1376\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m:\n\u001B[1;32m 1377\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclose()\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:318\u001B[0m, in \u001B[0;36mHTTPResponse.begin\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 316\u001B[0m \u001B[38;5;66;03m# read until we get a non-100 response\u001B[39;00m\n\u001B[1;32m 317\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m--> 318\u001B[0m version, status, reason \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_read_status\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 319\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m status \u001B[38;5;241m!=\u001B[39m CONTINUE:\n\u001B[1;32m 320\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/http/client.py:279\u001B[0m, in \u001B[0;36mHTTPResponse._read_status\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m 278\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_read_status\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[0;32m--> 279\u001B[0m line \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mstr\u001B[39m(\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreadline\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_MAXLINE\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m)\u001B[49m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124miso-8859-1\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[1;32m 280\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(line) \u001B[38;5;241m>\u001B[39m _MAXLINE:\n\u001B[1;32m 281\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m LineTooLong(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstatus line\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/socket.py:705\u001B[0m, in \u001B[0;36mSocketIO.readinto\u001B[0;34m(self, b)\u001B[0m\n\u001B[1;32m 703\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m 704\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 705\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sock\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrecv_into\u001B[49m\u001B[43m(\u001B[49m\u001B[43mb\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 706\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m timeout:\n\u001B[1;32m 707\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_timeout_occurred \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/ssl.py:1274\u001B[0m, in \u001B[0;36mSSLSocket.recv_into\u001B[0;34m(self, buffer, nbytes, flags)\u001B[0m\n\u001B[1;32m 1270\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m flags \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m 1271\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 1272\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m\n\u001B[1;32m 1273\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m)\n\u001B[0;32m-> 1274\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnbytes\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbuffer\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1275\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 1276\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39mrecv_into(buffer, nbytes, flags)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/ssl.py:1130\u001B[0m, in \u001B[0;36mSSLSocket.read\u001B[0;34m(self, len, buffer)\u001B[0m\n\u001B[1;32m 1128\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m 1129\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m buffer \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m-> 1130\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sslobj\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mread\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mlen\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbuffer\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1131\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 1132\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sslobj\u001B[38;5;241m.\u001B[39mread(\u001B[38;5;28mlen\u001B[39m)\n", - "\u001B[0;31mKeyboardInterrupt\u001B[0m: " + "\u001B[0;31mModuleNotFoundError\u001B[0m Traceback (most recent call last)", + "Cell \u001B[0;32mIn[3], line 8\u001B[0m\n\u001B[1;32m 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mautobuild\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m AgentCreator\n\u001B[1;32m 4\u001B[0m administrator \u001B[38;5;241m=\u001B[39m AgentCreator(\n\u001B[1;32m 5\u001B[0m task,\n\u001B[1;32m 6\u001B[0m config_path\u001B[38;5;241m=\u001B[39mconfig_path\n\u001B[1;32m 7\u001B[0m )\n\u001B[0;32m----> 8\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbuild\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdefault_llm_config\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 9\u001B[0m administrator\u001B[38;5;241m.\u001B[39mstart()\n\u001B[1;32m 10\u001B[0m administrator\u001B[38;5;241m.\u001B[39mclear_all()\n", + "File \u001B[0;32m~/LLM/autogen/autobuild/agent_creator.py:206\u001B[0m, in \u001B[0;36mAgentCreator.build\u001B[0;34m(self, default_llm_config, coding)\u001B[0m\n\u001B[1;32m 204\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m coding \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 205\u001B[0m api \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mOpenAIWrapper(config_list\u001B[38;5;241m=\u001B[39mconfig_list)\n\u001B[0;32m--> 206\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[43mapi\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 207\u001B[0m \u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m[\u001B[49m\u001B[43m{\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mrole\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43muser\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontent\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mCODING_PROMPT\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mformat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtask\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\u001B[43m}\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 208\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig_list\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m]\u001B[49m\n\u001B[1;32m 209\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241m.\u001B[39mchoices[\u001B[38;5;241m0\u001B[39m]\u001B[38;5;241m.\u001B[39mmessage\u001B[38;5;241m.\u001B[39mcontent\n\u001B[1;32m 210\u001B[0m coding \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m \u001B[38;5;28;01mif\u001B[39;00m resp \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mYES\u001B[39m\u001B[38;5;124m'\u001B[39m \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m 212\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m coding \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n", + "File \u001B[0;32m~/LLM/autogen/autogen/oai/client.py:230\u001B[0m, in \u001B[0;36mOpenAIWrapper.create\u001B[0;34m(self, **config)\u001B[0m\n\u001B[1;32m 227\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m seed \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 228\u001B[0m \u001B[38;5;66;03m# Try to get the response from cache\u001B[39;00m\n\u001B[1;32m 229\u001B[0m key \u001B[38;5;241m=\u001B[39m get_key(params)\n\u001B[0;32m--> 230\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mcache\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m 231\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m response \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 232\u001B[0m \u001B[38;5;66;03m# check the filter\u001B[39;00m\n\u001B[1;32m 233\u001B[0m pass_filter \u001B[38;5;241m=\u001B[39m filter_func \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m filter_func(context\u001B[38;5;241m=\u001B[39mcontext, response\u001B[38;5;241m=\u001B[39mresponse)\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/diskcache/core.py:1173\u001B[0m, in \u001B[0;36mCache.get\u001B[0;34m(self, key, default, read, expire_time, tag, retry)\u001B[0m\n\u001B[1;32m 1170\u001B[0m ((rowid, db_expire_time, db_tag, mode, filename, db_value),) \u001B[38;5;241m=\u001B[39m rows\n\u001B[1;32m 1172\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1173\u001B[0m value \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_disk\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfetch\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfilename\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdb_value\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mread\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1174\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mIOError\u001B[39;00m:\n\u001B[1;32m 1175\u001B[0m \u001B[38;5;66;03m# Key was deleted before we could retrieve result.\u001B[39;00m\n\u001B[1;32m 1176\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m default\n", + "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/diskcache/core.py:284\u001B[0m, in \u001B[0;36mDisk.fetch\u001B[0;34m(self, mode, filename, value, read)\u001B[0m\n\u001B[1;32m 282\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m pickle\u001B[38;5;241m.\u001B[39mload(reader)\n\u001B[1;32m 283\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 284\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mpickle\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload\u001B[49m\u001B[43m(\u001B[49m\u001B[43mio\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mBytesIO\u001B[49m\u001B[43m(\u001B[49m\u001B[43mvalue\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n", + "\u001B[0;31mModuleNotFoundError\u001B[0m: No module named 'openai.openai_object'" ] } ], @@ -232,8 +152,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-03T13:52:10.449037900Z", - "start_time": "2023-11-03T13:52:04.890203400Z" + "end_time": "2023-11-08T14:25:55.341502500Z", + "start_time": "2023-11-08T14:25:53.472544900Z" } }, "id": "7d52e3d9a1bf91cb" From 809e0732fe2a884cc7da7314c56f5fefb07568f4 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Thu, 9 Nov 2023 00:44:05 +0900 Subject: [PATCH 14/74] add example --- notebook/agentchat_autobuild.ipynb | 223 ++++++++++++++++++++++++++--- 1 file changed, 204 insertions(+), 19 deletions(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index b01be8853d6..6cd3ccbed05 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -62,8 +62,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-08T14:25:51.075351900Z", - "start_time": "2023-11-08T14:25:51.069815900Z" + "end_time": "2023-11-08T15:42:51.572203Z", + "start_time": "2023-11-08T15:42:51.568479900Z" } }, "id": "97221d43eb23768" @@ -92,8 +92,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-08T14:25:51.612271500Z", - "start_time": "2023-11-08T14:25:51.603397800Z" + "end_time": "2023-11-08T15:42:52.573162500Z", + "start_time": "2023-11-08T15:42:52.569650300Z" } }, "id": "2505f029423b21ab" @@ -118,22 +118,207 @@ "output_type": "stream", "text": [ "Initializing usable port...\n", - "57505 ports found.\n" + "57512 ports found.\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", + "\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", + "\n", + "Here is the Python code to accomplish this task:\n", + "\n", + "```python\n", + "import requests\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response and extract the paper details\n", + "xml_data = response.text\n", + "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", + "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", + "\n", + "# Print the paper details\n", + "print(\"Latest Paper about GPT-4 on arXiv:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Summary:\", paper_summary)\n", + "\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", + "\n", + "```\n", + "\n", + "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "\n", + "When you run the code, it will print the title and summary of the latest paper about GPT-4 on arXiv. It will also analyze the paper's summary and identify potential applications in software based on predefined keywords.\n", + "\n", + "Let me know if you need any further assistance!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" ] }, { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'openai.openai_object'", - "output_type": "error", - "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mModuleNotFoundError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[3], line 8\u001B[0m\n\u001B[1;32m 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mautobuild\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m AgentCreator\n\u001B[1;32m 4\u001B[0m administrator \u001B[38;5;241m=\u001B[39m AgentCreator(\n\u001B[1;32m 5\u001B[0m task,\n\u001B[1;32m 6\u001B[0m config_path\u001B[38;5;241m=\u001B[39mconfig_path\n\u001B[1;32m 7\u001B[0m )\n\u001B[0;32m----> 8\u001B[0m \u001B[43madministrator\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbuild\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdefault_llm_config\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 9\u001B[0m administrator\u001B[38;5;241m.\u001B[39mstart()\n\u001B[1;32m 10\u001B[0m administrator\u001B[38;5;241m.\u001B[39mclear_all()\n", - "File \u001B[0;32m~/LLM/autogen/autobuild/agent_creator.py:206\u001B[0m, in \u001B[0;36mAgentCreator.build\u001B[0;34m(self, default_llm_config, coding)\u001B[0m\n\u001B[1;32m 204\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m coding \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 205\u001B[0m api \u001B[38;5;241m=\u001B[39m autogen\u001B[38;5;241m.\u001B[39mOpenAIWrapper(config_list\u001B[38;5;241m=\u001B[39mconfig_list)\n\u001B[0;32m--> 206\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[43mapi\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcreate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m 207\u001B[0m \u001B[43m \u001B[49m\u001B[43mmessages\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m[\u001B[49m\u001B[43m{\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mrole\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43muser\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mcontent\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mCODING_PROMPT\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mformat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtask\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtask\u001B[49m\u001B[43m)\u001B[49m\u001B[43m}\u001B[49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m 208\u001B[0m \u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mconfig_list\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m]\u001B[49m\n\u001B[1;32m 209\u001B[0m \u001B[43m \u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241m.\u001B[39mchoices[\u001B[38;5;241m0\u001B[39m]\u001B[38;5;241m.\u001B[39mmessage\u001B[38;5;241m.\u001B[39mcontent\n\u001B[1;32m 210\u001B[0m coding \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m \u001B[38;5;28;01mif\u001B[39;00m resp \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mYES\u001B[39m\u001B[38;5;124m'\u001B[39m \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m 212\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m coding \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n", - "File \u001B[0;32m~/LLM/autogen/autogen/oai/client.py:230\u001B[0m, in \u001B[0;36mOpenAIWrapper.create\u001B[0;34m(self, **config)\u001B[0m\n\u001B[1;32m 227\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m seed \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 228\u001B[0m \u001B[38;5;66;03m# Try to get the response from cache\u001B[39;00m\n\u001B[1;32m 229\u001B[0m key \u001B[38;5;241m=\u001B[39m get_key(params)\n\u001B[0;32m--> 230\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[43mcache\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget\u001B[49m\u001B[43m(\u001B[49m\u001B[43mkey\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m 231\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m response \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 232\u001B[0m \u001B[38;5;66;03m# check the filter\u001B[39;00m\n\u001B[1;32m 233\u001B[0m pass_filter \u001B[38;5;241m=\u001B[39m filter_func \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mor\u001B[39;00m filter_func(context\u001B[38;5;241m=\u001B[39mcontext, response\u001B[38;5;241m=\u001B[39mresponse)\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/diskcache/core.py:1173\u001B[0m, in \u001B[0;36mCache.get\u001B[0;34m(self, key, default, read, expire_time, tag, retry)\u001B[0m\n\u001B[1;32m 1170\u001B[0m ((rowid, db_expire_time, db_tag, mode, filename, db_value),) \u001B[38;5;241m=\u001B[39m rows\n\u001B[1;32m 1172\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1173\u001B[0m value \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_disk\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfetch\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfilename\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdb_value\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mread\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 1174\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mIOError\u001B[39;00m:\n\u001B[1;32m 1175\u001B[0m \u001B[38;5;66;03m# Key was deleted before we could retrieve result.\u001B[39;00m\n\u001B[1;32m 1176\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m default\n", - "File \u001B[0;32m~/anaconda3/envs/llm/lib/python3.10/site-packages/diskcache/core.py:284\u001B[0m, in \u001B[0;36mDisk.fetch\u001B[0;34m(self, mode, filename, value, read)\u001B[0m\n\u001B[1;32m 282\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m pickle\u001B[38;5;241m.\u001B[39mload(reader)\n\u001B[1;32m 283\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 284\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mpickle\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload\u001B[49m\u001B[43m(\u001B[49m\u001B[43mio\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mBytesIO\u001B[49m\u001B[43m(\u001B[49m\u001B[43mvalue\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n", - "\u001B[0;31mModuleNotFoundError\u001B[0m: No module named 'openai.openai_object'" + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "exitcode: 1 (execution failed)\n", + "Code output: \n", + "Traceback (most recent call last):\n", + " File \"\", line 10, in \n", + " paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", + "IndexError: list index out of range\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", + "\n", + "I apologize for the error. It seems that the code failed to extract the paper details from the XML response. This could be due to changes in the XML structure or the absence of the expected tags.\n", + "\n", + "To resolve this issue, we can use the `feedparser` library to parse the XML response and extract the paper details. Here's the updated code:\n", + "\n", + "```python\n", + "import requests\n", + "import feedparser\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response using feedparser\n", + "feed = feedparser.parse(response.text)\n", + "\n", + "# Extract the paper details\n", + "if len(feed.entries) > 0:\n", + " paper_title = feed.entries[0].title\n", + " paper_summary = feed.entries[0].summary\n", + "else:\n", + " print(\"No papers found for GPT-4 on arXiv.\")\n", + " exit()\n", + "\n", + "# Print the paper details\n", + "print(\"Latest Paper about GPT-4 on arXiv:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Summary:\", paper_summary)\n", + "\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", + "```\n", + "\n", + "Please make sure you have the `requests` and `feedparser` libraries installed (`pip install requests feedparser`) before running the code.\n", + "\n", + "This updated code should be able to extract the paper details correctly. Let me know if you encounter any further issues!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Latest Paper about GPT-4 on arXiv:\n", + "Title: Can LLMs like GPT-4 outperform traditional AI tools in dementia\n", + " diagnosis? Maybe, but not today\n", + "Summary: Recent investigations show that large language models (LLMs), specifically\n", + "GPT-4, not only have remarkable capabilities in common Natural Language\n", + "Processing (NLP) tasks but also exhibit human-level performance on various\n", + "professional and academic benchmarks. However, whether GPT-4 can be directly\n", + "used in practical applications and replace traditional artificial intelligence\n", + "(AI) tools in specialized domains requires further experimental validation. In\n", + "this paper, we explore the potential of LLMs such as GPT-4 to outperform\n", + "traditional AI tools in dementia diagnosis. Comprehensive comparisons between\n", + "GPT-4 and traditional AI tools are conducted to examine their diagnostic\n", + "accuracy in a clinical setting. Experimental results on two real clinical\n", + "datasets show that, although LLMs like GPT-4 demonstrate potential for future\n", + "advancements in dementia diagnosis, they currently do not surpass the\n", + "performance of traditional AI tools. The interpretability and faithfulness of\n", + "GPT-4 are also evaluated by comparison with real doctors. We discuss the\n", + "limitations of GPT-4 in its current state and propose future research\n", + "directions to enhance GPT-4 in dementia diagnosis.\n", + "\n", + "Potential Applications in Software:\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager\u001B[0m (to chat_manager):\n", + "\n", + "Based on the summary of the latest paper about GPT-4 on arXiv, the potential applications in software could include:\n", + "\n", + "- Natural Language Processing: GPT-4 has remarkable capabilities in common NLP tasks, such as text classification, sentiment analysis, and named entity recognition.\n", + "- Text Generation: GPT-4 can generate human-like text, which can be useful in various software applications, such as content generation, chatbots, and virtual assistants.\n", + "- Chatbots: GPT-4 can be used to develop more advanced and conversational chatbots that can understand and respond to user queries in a more natural and human-like manner.\n", + "- Language Translation: GPT-4's language modeling capabilities can be leveraged for language translation tasks, enabling more accurate and fluent translations between different languages.\n", + "- Sentiment Analysis: GPT-4 can be used to analyze and understand the sentiment expressed in text, which can be valuable in applications such as social media monitoring, customer feedback analysis, and market research.\n", + "\n", + "These are just a few potential applications based on the summary of the paper. The actual applications of GPT-4 in software may vary depending on specific use cases and requirements.\n", + "\n", + "Let me know if there's anything else I can help you with!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", + "\n", + "Great! It seems like you have a good understanding of the potential applications of GPT-4 in software based on the summary of the paper. These applications can indeed be valuable in various domains, including natural language processing, text generation, chatbots, language translation, and sentiment analysis.\n", + "\n", + "If you have any more questions or need further assistance, feel free to ask.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mProduct_manager\u001B[0m (to chat_manager):\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" ] } ], @@ -152,8 +337,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-08T14:25:55.341502500Z", - "start_time": "2023-11-08T14:25:53.472544900Z" + "end_time": "2023-11-08T15:43:38.964405Z", + "start_time": "2023-11-08T15:42:53.646145900Z" } }, "id": "7d52e3d9a1bf91cb" From faa54ad1f1a3e8a28822a8542b05fa41433a9548 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 10 Nov 2023 04:06:28 +0900 Subject: [PATCH 15/74] add load_config, save_config, and add task in build and start --- autobuild/agent_creator.py | 104 ++++++++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 25 deletions(-) diff --git a/autobuild/agent_creator.py b/autobuild/agent_creator.py index 44eb9a95ac4..2540a2bf9d2 100644 --- a/autobuild/agent_creator.py +++ b/autobuild/agent_creator.py @@ -2,6 +2,9 @@ import time import subprocess as sp import socket +import os +import json +import hashlib from typing import * @@ -9,9 +12,6 @@ class AgentCreator: """ Descriptions """ - host: str - task: str - config_path: str open_ports: List[str] = [] agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} openai_server_name: str = 'openai' @@ -34,22 +34,24 @@ class AgentCreator: def __init__( self, - task: str, host: str = 'localhost', config_path: str = 'OAI_CONFIG_LIST', + build_config_path: Optional[str] = './.build_cache', endpoint_building_timeout: Optional[int] = 180 ): """ Args: - task: description of a task. endpoint_building_timeout: timeout for building up an endpoint server. config_path: path of the OpenAI api configs. + build_config_path: path of the build configs. host: endpoint host. """ - self.task = task self.endpoint_building_timeout = endpoint_building_timeout self.config_path = config_path self.host = host + self.build_config_path = build_config_path + if not os.path.exists(build_config_path): + os.makedirs(build_config_path) print('Initializing usable port...') for port in range(8000, 65535): @@ -69,9 +71,6 @@ def _is_port_open(host, port): except OSError: return False - def set_task(self, task: str): - self.task = task - def create_agent( self, agent_name: str, @@ -177,7 +176,7 @@ def clear_agent( self.agent_procs[server_id][0].terminate() self.open_ports.append(server_id.split('_')[-1]) - def clear_all(self): + def clear_all_agents(self): """ Clear all cached agents. """ @@ -186,25 +185,59 @@ def clear_all(self): def build( self, + task: str, default_llm_config: dict, - coding: bool = None + coding: bool = None, + use_cache: Optional[bool] = True ): - # TODO: not completed. + use_api = False + build_configs = {} + + if use_cache: + build_configs = self._load_config(task) + + if build_configs == {}: + use_api = True + else: + agent_configs = build_configs['agent_configs'] + coding = build_configs['coding'] + self.manager_system_message = build_configs['manager_system_message'] + config_list = autogen.config_list_from_json( self.config_path, filter_dict={ 'model': ['gpt-4'] } ) - agent_configs = [('Coder_gpt-35', 'gpt-3.5-turbo'), ('Product_manager', 'gpt-3.5-turbo')] + build_manager = autogen.OpenAIWrapper(config_list=config_list) + + # TODO: use the build manager to decide what agent should be created, + # and generate system message for each agent and group chat manager. + if use_api: + pass + + agent_configs = [ + { + 'name': 'Coder_gpt_35', + 'model': 'gpt-3.5-turbo', + 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + }, + { + 'name': 'Product_manager', + 'model': 'gpt-3.5-turbo', + 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + } + ] for agent_config in agent_configs: - self.create_agent(agent_config[0], agent_config[1], default_llm_config) + self.create_agent(agent_config['name'], + agent_config['model'], + default_llm_config, + system_message=agent_config['system_message']) if coding is None: - api = autogen.OpenAIWrapper(config_list=config_list) - resp = api.create( - messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.task)}] + resp = build_manager.create( + messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=task)}] ).choices[0].message.content coding = True if resp == 'YES' else False @@ -216,14 +249,35 @@ def build( human_input_mode="TERMINATE" ) else: - self.initiate_agent_name = agent_configs[0][0] + self.initiate_agent_name = agent_configs[0]['name'] self.group_chat_manager_config = default_llm_config.copy() self.group_chat_manager_config['config_list'] = config_list self.manager_system_message = 'Group chat manager.' + # TODO: save config. + save_config = { + 'agent_configs': agent_configs, + 'manager_system_message': self.manager_system_message, + 'coding': coding + } + self._save_config(task, save_config) + + def _save_config(self, task: str, config: dict): + filename = hashlib.md5(task.encode('utf-8')).hexdigest() + json.dump(config, open(f'{self.build_config_path}/{filename}.json', 'w'), indent=4) + + def _load_config(self, task: str): + filename = hashlib.md5(task.encode('utf-8')).hexdigest() + filepath = f'{self.build_config_path}/{filename}.json' + if os.path.isfile(filepath): + return json.load(open(filepath)) + else: + return {} + def start( self, + task: str, max_round: Optional[int] = 12, init_messages: Optional[List[dict]] = [] ): @@ -231,10 +285,10 @@ def start( Descriptions Args: + task: description of a task. max_round: the maximum number of rounds. init_messages: input messages before the task start. This can be the chat history from other group chat or some preliminary of the task. - initiate_agent_name: the name of an agent use to initialize the group chat. """ agent_list = [agent for agent, _ in self.agent_procs_assign.values()] if self.user_proxy is not None: @@ -246,11 +300,11 @@ def start( system_message=self.manager_system_message) if self.initiate_agent_name == "user" and self.user_proxy is not None: - self.user_proxy.initiate_chat(manager, message=self.task) + self.user_proxy.initiate_chat(manager, message=task) else: for agent in agent_list: if self.initiate_agent_name == agent.name(): - agent.initiate_chat(manager, message=self.task) + agent.initiate_chat(manager, message=task) if __name__ == '__main__': @@ -258,12 +312,12 @@ def start( default_llm_config = { 'temperature': 0 } + task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." administrator = AgentCreator( - task="Find a latest paper about gpt-4 on arxiv and find its potential applications in software.", config_path=config_path ) - administrator.build(default_llm_config) - administrator.start() - administrator.clear_all() + administrator.build(task, default_llm_config) + administrator.start(task) + administrator.clear_all_agents() From 987bb330ae8d1e9f652ebbb038220cea6d433a5c Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 10 Nov 2023 04:08:27 +0900 Subject: [PATCH 16/74] modify notebook --- notebook/agentchat_autobuild.ipynb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 6cd3ccbed05..2d926ffdd74 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -327,12 +327,11 @@ "\n", "\n", "administrator = AgentCreator(\n", - " task,\n", - " config_path=config_path\n", - ")\n", - "administrator.build(default_llm_config)\n", - "administrator.start()\n", - "administrator.clear_all()\n" + " config_path=config_path\n", + " )\n", + "administrator.build(task, default_llm_config)\n", + "administrator.start(task)\n", + "administrator.clear_all_agents()" ], "metadata": { "collapsed": false, From 6e07a8dbcaa59c29c0700b568ea1de73b4694719 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 11 Nov 2023 00:04:17 +0900 Subject: [PATCH 17/74] rewrite save and load function; update notebook --- autobuild/agent_creator.py | 147 +++++++------ notebook/agentchat_autobuild.ipynb | 326 ++++++++++++++++++++++++++--- 2 files changed, 385 insertions(+), 88 deletions(-) diff --git a/autobuild/agent_creator.py b/autobuild/agent_creator.py index 2540a2bf9d2..043a342339b 100644 --- a/autobuild/agent_creator.py +++ b/autobuild/agent_creator.py @@ -15,7 +15,6 @@ class AgentCreator: open_ports: List[str] = [] agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} openai_server_name: str = 'openai' - endpoint_building_timeout: Optional[int] agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} max_tokens: int = 945 @@ -23,6 +22,10 @@ class AgentCreator: group_chat_manager_config: dict = None initiate_agent_name: str = 'user' manager_system_message: str = 'Group chat manager.' + agent_configs: List[Dict] = None + coding: bool = None + default_llm_config: Dict = None + building_task: str = None CODING_PROMPT: str = '''Does the following task need programming (i.e., access external API or tool by coding) to solve? @@ -36,22 +39,17 @@ def __init__( self, host: str = 'localhost', config_path: str = 'OAI_CONFIG_LIST', - build_config_path: Optional[str] = './.build_cache', endpoint_building_timeout: Optional[int] = 180 ): """ Args: - endpoint_building_timeout: timeout for building up an endpoint server. - config_path: path of the OpenAI api configs. - build_config_path: path of the build configs. host: endpoint host. + config_path: path of the OpenAI api configs. + endpoint_building_timeout: timeout for building up an endpoint server. """ self.endpoint_building_timeout = endpoint_building_timeout self.config_path = config_path self.host = host - self.build_config_path = build_config_path - if not os.path.exists(build_config_path): - os.makedirs(build_config_path) print('Initializing usable port...') for port in range(8000, 65535): @@ -61,7 +59,7 @@ def __init__( @staticmethod def _is_port_open(host, port): - """Check if a port is open.""" + """Check if a tcp port is open.""" try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(10) @@ -80,7 +78,10 @@ def create_agent( world_size: Optional[int] = 1 ) -> autogen.AssistantAgent: """ - Descriptions + Create a group chat agent. + + If the agent rely on an open-source model, this function will automatically set up an endpoint for that agent. + The API address of that endpoint will be "localhost:{free port}". Args: agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...) @@ -157,7 +158,7 @@ def clear_agent( recycle_endpoint: bool = True ): """ - Descriptions + Clear a specific agent by name. Args: agent_name: the name of agent. @@ -185,23 +186,24 @@ def clear_all_agents(self): def build( self, - task: str, - default_llm_config: dict, + building_task: str = None, + default_llm_config: Dict = None, coding: bool = None, - use_cache: Optional[bool] = True + cache_configs: Optional[Dict] = None ): use_api = False - build_configs = {} - if use_cache: - build_configs = self._load_config(task) - - if build_configs == {}: + if cache_configs is None: use_api = True + self.building_task = building_task + self.default_llm_config = default_llm_config.copy() + self.coding = coding else: - agent_configs = build_configs['agent_configs'] - coding = build_configs['coding'] - self.manager_system_message = build_configs['manager_system_message'] + self.building_task = cache_configs['building_task'] + self.default_llm_config = cache_configs['default_llm_config'] + self.coding = cache_configs['coding'] + self.agent_configs = cache_configs['agent_configs'] + self.manager_system_message = cache_configs['manager_system_message'] config_list = autogen.config_list_from_json( self.config_path, @@ -214,34 +216,34 @@ def build( # TODO: use the build manager to decide what agent should be created, # and generate system message for each agent and group chat manager. if use_api: - pass - - agent_configs = [ - { - 'name': 'Coder_gpt_35', - 'model': 'gpt-3.5-turbo', - 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE - }, - { - 'name': 'Product_manager', - 'model': 'gpt-3.5-turbo', - 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE - } - ] - - for agent_config in agent_configs: + # after this process completed, we should obtain a following list and a manager_system_config. + self.agent_configs = [ + { + 'name': 'Coder_gpt_35', + 'model': 'gpt-3.5-turbo', + 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + }, + { + 'name': 'Product_manager', + 'model': 'gpt-3.5-turbo', + 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + } + ] + self.manager_system_message = 'Group chat manager.' + + for agent_config in self.agent_configs: self.create_agent(agent_config['name'], agent_config['model'], - default_llm_config, + self.default_llm_config, system_message=agent_config['system_message']) - if coding is None: + if self.coding is None: resp = build_manager.create( - messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=task)}] + messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.building_task)}] ).choices[0].message.content - coding = True if resp == 'YES' else False + self.coding = True if resp == 'YES' else False - if coding is True: + if self.coding is True: self.user_proxy = autogen.UserProxyAgent( name="User_proxy", system_message="A human admin.", @@ -249,31 +251,53 @@ def build( human_input_mode="TERMINATE" ) else: - self.initiate_agent_name = agent_configs[0]['name'] + self.initiate_agent_name = self.agent_configs[0]['name'] - self.group_chat_manager_config = default_llm_config.copy() + self.group_chat_manager_config = self.default_llm_config.copy() self.group_chat_manager_config['config_list'] = config_list - self.manager_system_message = 'Group chat manager.' - # TODO: save config. - save_config = { - 'agent_configs': agent_configs, + def save( + self, + filepath: Optional[str] = None + ) -> str: + """ + Save building configs. If the filepath is not specific, this function will create a filename by encrypt the + building_task string by md5 with "save_config_" prefix, and save config to the local path. + + Args: + filepath: save path. + + Return: + filepath: saved path. + """ + if filepath is None: + filepath = f'./save_config_{hashlib.md5(self.building_task.encode("utf-8")).hexdigest()}.json' + json.dump({ + 'building_task': self.building_task, + 'agent_configs': self.agent_configs, 'manager_system_message': self.manager_system_message, - 'coding': coding - } - self._save_config(task, save_config) + 'coding': self.coding, + 'default_llm_config': self.default_llm_config + }, open(filepath, 'w'), indent=4) + print(f'Building config saved to {filepath}') + + return filepath - def _save_config(self, task: str, config: dict): - filename = hashlib.md5(task.encode('utf-8')).hexdigest() - json.dump(config, open(f'{self.build_config_path}/{filename}.json', 'w'), indent=4) + def load( + self, + filepath: str + ): + """ + Load building configs and call the build function to complete building without calling online LLMs' api. - def _load_config(self, task: str): - filename = hashlib.md5(task.encode('utf-8')).hexdigest() - filepath = f'{self.build_config_path}/{filename}.json' + Args: + filepath: filepath for the save config. + """ if os.path.isfile(filepath): - return json.load(open(filepath)) + cache_configs = json.load(open(filepath)) + self.build(cache_configs=cache_configs) else: - return {} + raise FileNotFoundError(f"Config file {filepath} does not exist.") def start( self, @@ -282,7 +306,7 @@ def start( init_messages: Optional[List[dict]] = [] ): """ - Descriptions + Start a group chat task solving process with built config. Args: task: description of a task. @@ -318,6 +342,7 @@ def start( config_path=config_path ) administrator.build(task, default_llm_config) + administrator.save() administrator.start(task) administrator.clear_all_agents() diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 2d926ffdd74..faccb1463b9 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -62,12 +62,38 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-08T15:42:51.572203Z", - "start_time": "2023-11-08T15:42:51.568479900Z" + "end_time": "2023-11-10T15:02:36.810368900Z", + "start_time": "2023-11-10T15:02:36.799309700Z" } }, "id": "97221d43eb23768" }, + { + "cell_type": "markdown", + "source": [ + "(Optional) you can specify a building task in natural language to let build manager (a LLM) build a specific group of agent for your task (e.g., \"Create a product_manager and coder for me, use gpt-3.5-turbo as backbone.\"), or just use the same task as building task and let build manager to decide which agent should be built for your task." + ], + "metadata": { + "collapsed": false + }, + "id": "2e6a655fb6618324" + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "building_task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T15:02:37.598759400Z", + "start_time": "2023-11-10T15:02:37.587165900Z" + } + }, + "id": "68315f6ec912c58a" + }, { "cell_type": "markdown", "source": [ @@ -81,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "outputs": [], "source": [ "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", @@ -92,8 +118,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-08T15:42:52.573162500Z", - "start_time": "2023-11-08T15:42:52.569650300Z" + "end_time": "2023-11-10T15:02:38.968684300Z", + "start_time": "2023-11-10T15:02:38.966674300Z" } }, "id": "2505f029423b21ab" @@ -111,19 +137,20 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initializing usable port...\n", - "57512 ports found.\n", + "57510 ports found.\n", "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", + "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", "\n", "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", "\n", @@ -191,7 +218,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", + "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", "\n", "I apologize for the error. It seems that the code failed to extract the paper details from the XML response. This could be due to changes in the XML structure or the absence of the expected tags.\n", "\n", @@ -278,15 +305,16 @@ "\n", "Potential Applications in Software:\n", "\n", + "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mProduct_manager\u001B[0m (to chat_manager):\n", "\n", "Based on the summary of the latest paper about GPT-4 on arXiv, the potential applications in software could include:\n", "\n", "- Natural Language Processing: GPT-4 has remarkable capabilities in common NLP tasks, such as text classification, sentiment analysis, and named entity recognition.\n", - "- Text Generation: GPT-4 can generate human-like text, which can be useful in various software applications, such as content generation, chatbots, and virtual assistants.\n", + "- Text Generation: GPT-4 can generate coherent and contextually relevant text, which can be useful in various software applications, such as content generation, chatbots, and virtual assistants.\n", "- Chatbots: GPT-4 can be used to develop more advanced and conversational chatbots that can understand and respond to user queries in a more natural and human-like manner.\n", - "- Language Translation: GPT-4's language modeling capabilities can be leveraged for language translation tasks, enabling more accurate and fluent translations between different languages.\n", + "- Language Translation: GPT-4's language modeling capabilities can be leveraged for language translation tasks, enabling more accurate and context-aware translations.\n", "- Sentiment Analysis: GPT-4 can be used to analyze and understand the sentiment expressed in text, which can be valuable in applications such as social media monitoring, customer feedback analysis, and market research.\n", "\n", "These are just a few potential applications based on the summary of the paper. The actual applications of GPT-4 in software may vary depending on specific use cases and requirements.\n", @@ -294,28 +322,277 @@ "Let me know if there's anything else I can help you with!\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt-35\u001B[0m (to chat_manager):\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "from autobuild import AgentCreator\n", + "\n", + "administrator = AgentCreator(config_path=config_path)\n", + "administrator.build(task, default_llm_config)\n", + "administrator.start(task)\n", + "administrator.clear_all_agents()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T15:02:42.990993500Z", + "start_time": "2023-11-10T15:02:40.196779700Z" + } + }, + "id": "7d52e3d9a1bf91cb" + }, + { + "cell_type": "markdown", + "source": [ + "## (Optional) Save configs\n", + "\n", + "You can save the building config by calling `save()` function. This function will save all necessary information for building all group chat agents. You can provide a specific filename for this function. If not, it will save config to the local path with a generated filename 'save_config_TASK_MD5.json'." + ], + "metadata": { + "collapsed": false + }, + "id": "bbb098638a086898" + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Building config saved to ./save_config_5e043e06bd30f3593fdaf15e8b594002.json\n" + ] + } + ], + "source": [ + "saved_path = administrator.save()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-10T15:02:45.006608600Z", + "start_time": "2023-11-10T15:02:44.999076300Z" + } + }, + "id": "e4b88a5d482ceba4" + }, + { + "cell_type": "markdown", + "source": [ + "Once you save the building config, you can call `load()` function with the path of the saved config to skip the building process (note that building process will call online LLMs' api)." + ], + "metadata": { + "collapsed": false + }, + "id": "a35620c10ee42be" + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Initializing usable port...\n", + "115020 ports found.\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", + "\n", + "Here is the Python code to accomplish this task:\n", + "\n", + "```python\n", + "import requests\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response and extract the paper details\n", + "xml_data = response.text\n", + "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", + "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", + "\n", + "# Print the paper details\n", + "print(\"Latest Paper about GPT-4 on arXiv:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Summary:\", paper_summary)\n", + "\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", + "\n", + "```\n", "\n", - "Great! It seems like you have a good understanding of the potential applications of GPT-4 in software based on the summary of the paper. These applications can indeed be valuable in various domains, including natural language processing, text generation, chatbots, language translation, and sentiment analysis.\n", + "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", "\n", - "If you have any more questions or need further assistance, feel free to ask.\n", + "When you run the code, it will print the title and summary of the latest paper about GPT-4 on arXiv. It will also analyze the paper's summary and identify potential applications in software based on predefined keywords.\n", + "\n", + "Let me know if you need any further assistance!\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", "\n", + "exitcode: 1 (execution failed)\n", + "Code output: \n", + "Traceback (most recent call last):\n", + " File \"\", line 10, in \n", + " paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", + "IndexError: list index out of range\n", "\n", "\n", "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\n", + "I apologize for the error. It seems that the code failed to extract the paper details from the XML response. This could be due to changes in the XML structure or the absence of the expected tags.\n", + "\n", + "To resolve this issue, we can use the `feedparser` library to parse the XML response and extract the paper details. Here's the updated code:\n", + "\n", + "```python\n", + "import requests\n", + "import feedparser\n", + "\n", + "# Search for papers related to GPT-4 on arXiv\n", + "search_query = \"GPT-4\"\n", + "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", + "response = requests.get(api_url)\n", + "\n", + "# Parse the XML response using feedparser\n", + "feed = feedparser.parse(response.text)\n", + "\n", + "# Extract the paper details\n", + "if len(feed.entries) > 0:\n", + " paper_title = feed.entries[0].title\n", + " paper_summary = feed.entries[0].summary\n", + "else:\n", + " print(\"No papers found for GPT-4 on arXiv.\")\n", + " exit()\n", + "\n", + "# Print the paper details\n", + "print(\"Latest Paper about GPT-4 on arXiv:\")\n", + "print(\"Title:\", paper_title)\n", + "print(\"Summary:\", paper_summary)\n", + "\n", + "# Analyze the potential applications in software\n", + "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", + "print(\"\\nPotential Applications in Software:\")\n", + "for application in potential_applications:\n", + " if application.lower() in paper_summary.lower():\n", + " print(\"-\", application)\n", + "```\n", + "\n", + "Please make sure you have the `requests` and `feedparser` libraries installed (`pip install requests feedparser`) before running the code.\n", + "\n", + "This updated code should be able to extract the paper details correctly. Let me know if you encounter any further issues!\n", + "\n", + "--------------------------------------------------------------------------------\n", "\u001B[31m\n", ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Latest Paper about GPT-4 on arXiv:\n", + "Title: Can LLMs like GPT-4 outperform traditional AI tools in dementia\n", + " diagnosis? Maybe, but not today\n", + "Summary: Recent investigations show that large language models (LLMs), specifically\n", + "GPT-4, not only have remarkable capabilities in common Natural Language\n", + "Processing (NLP) tasks but also exhibit human-level performance on various\n", + "professional and academic benchmarks. However, whether GPT-4 can be directly\n", + "used in practical applications and replace traditional artificial intelligence\n", + "(AI) tools in specialized domains requires further experimental validation. In\n", + "this paper, we explore the potential of LLMs such as GPT-4 to outperform\n", + "traditional AI tools in dementia diagnosis. Comprehensive comparisons between\n", + "GPT-4 and traditional AI tools are conducted to examine their diagnostic\n", + "accuracy in a clinical setting. Experimental results on two real clinical\n", + "datasets show that, although LLMs like GPT-4 demonstrate potential for future\n", + "advancements in dementia diagnosis, they currently do not surpass the\n", + "performance of traditional AI tools. The interpretability and faithfulness of\n", + "GPT-4 are also evaluated by comparison with real doctors. We discuss the\n", + "limitations of GPT-4 in its current state and propose future research\n", + "directions to enhance GPT-4 in dementia diagnosis.\n", "\n", + "Potential Applications in Software:\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mProduct_manager\u001B[0m (to chat_manager):\n", + "\n", + "Based on the summary of the latest paper about GPT-4 on arXiv, the potential applications in software could include:\n", + "\n", + "- Natural Language Processing: GPT-4 has remarkable capabilities in common NLP tasks, such as text classification, sentiment analysis, and named entity recognition.\n", + "- Text Generation: GPT-4 can generate coherent and contextually relevant text, which can be useful in various software applications, such as content generation, chatbots, and virtual assistants.\n", + "- Chatbots: GPT-4 can be used to develop more advanced and conversational chatbots that can understand and respond to user queries in a more natural and human-like manner.\n", + "- Language Translation: GPT-4's language modeling capabilities can be leveraged for language translation tasks, enabling more accurate and context-aware translations.\n", + "- Sentiment Analysis: GPT-4 can be used to analyze and understand the sentiment expressed in text, which can be valuable in applications such as social media monitoring, customer feedback analysis, and market research.\n", + "\n", + "These are just a few potential applications based on the summary of the paper. The actual applications of GPT-4 in software may vary depending on specific use cases and requirements.\n", + "\n", + "Let me know if there's anything else I can help you with!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", + "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n" @@ -323,24 +600,19 @@ } ], "source": [ - "from autobuild import AgentCreator\n", - "\n", - "\n", - "administrator = AgentCreator(\n", - " config_path=config_path\n", - " )\n", - "administrator.build(task, default_llm_config)\n", - "administrator.start(task)\n", - "administrator.clear_all_agents()" + "new_administrator = AgentCreator(config_path=config_path)\n", + "new_administrator.load(saved_path)\n", + "new_administrator.start(task)\n", + "new_administrator.clear_all_agents()" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-08T15:43:38.964405Z", - "start_time": "2023-11-08T15:42:53.646145900Z" + "end_time": "2023-11-10T15:03:27.550852700Z", + "start_time": "2023-11-10T15:03:15.297577500Z" } }, - "id": "7d52e3d9a1bf91cb" + "id": "34addd498e5ab174" }, { "cell_type": "code", @@ -350,7 +622,7 @@ "metadata": { "collapsed": false }, - "id": "4ca05bc74723e163" + "id": "788ab82ea53d03b4" } ], "metadata": { From cafe4c429a0a56ae0c9e9f90aa0741aff4ccd477 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 11 Nov 2023 01:30:05 +0900 Subject: [PATCH 18/74] update description --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index faccb1463b9..e82cbc961d3 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -359,7 +359,7 @@ "source": [ "## (Optional) Save configs\n", "\n", - "You can save the building config by calling `save()` function. This function will save all necessary information for building all group chat agents. You can provide a specific filename for this function. If not, it will save config to the local path with a generated filename 'save_config_TASK_MD5.json'." + "You can save the building config by calling `save()` function. This function will save all necessary information for building all group chat agents. You can provide a specific filename, otherwise, it will save config to the current path with a generated filename 'save_config_TASK_MD5.json'." ], "metadata": { "collapsed": false From 57940bfb5b49f3e16283b3fabf4f432c55dc7fc5 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 11 Nov 2023 01:33:42 +0900 Subject: [PATCH 19/74] update description --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index e82cbc961d3..954b7034de3 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -110,7 +110,7 @@ "execution_count": 3, "outputs": [], "source": [ - "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", + "config_path = 'YOUR CONFIG PATH OR ENVIRONMENT'\n", "default_llm_config = {\n", " 'temperature': 0\n", "}" From 4a50ba7a1624ae64b211bb628f6611accd1d4cfc Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 11 Nov 2023 01:36:31 +0900 Subject: [PATCH 20/74] update description --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 954b7034de3..01d63bc2d8d 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -341,7 +341,7 @@ "from autobuild import AgentCreator\n", "\n", "administrator = AgentCreator(config_path=config_path)\n", - "administrator.build(task, default_llm_config)\n", + "administrator.build(building_task, default_llm_config)\n", "administrator.start(task)\n", "administrator.clear_all_agents()" ], From 7bdfe848490bd2f4859fe8d88c92ad82805f98c8 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 11 Nov 2023 01:42:49 +0900 Subject: [PATCH 21/74] change member variable of agent_creator.py --- autobuild/agent_creator.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/autobuild/agent_creator.py b/autobuild/agent_creator.py index 043a342339b..e197fa5152f 100644 --- a/autobuild/agent_creator.py +++ b/autobuild/agent_creator.py @@ -12,21 +12,9 @@ class AgentCreator: """ Descriptions """ - open_ports: List[str] = [] - agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} openai_server_name: str = 'openai' - agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} max_tokens: int = 945 - user_proxy: autogen.UserProxyAgent = None - group_chat_manager_config: dict = None - initiate_agent_name: str = 'user' - manager_system_message: str = 'Group chat manager.' - agent_configs: List[Dict] = None - coding: bool = None - default_llm_config: Dict = None - building_task: str = None - CODING_PROMPT: str = '''Does the following task need programming (i.e., access external API or tool by coding) to solve? @@ -47,6 +35,19 @@ def __init__( config_path: path of the OpenAI api configs. endpoint_building_timeout: timeout for building up an endpoint server. """ + self.user_proxy: autogen.UserProxyAgent = None + self.group_chat_manager_config: dict = None + self.initiate_agent_name: str = 'user' + self.manager_system_message: str = 'Group chat manager.' + self.agent_configs: List[Dict] = None + self.coding: bool = None + self.default_llm_config: Dict = None + self.building_task: str = None + + self.open_ports: List[str] = [] + self.agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} + self.agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} + self.endpoint_building_timeout = endpoint_building_timeout self.config_path = config_path self.host = host From 8da1eb3566d2ab29a42d17390deac19bb667f7be Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 11 Nov 2023 01:45:20 +0900 Subject: [PATCH 22/74] update notebook --- notebook/agentchat_autobuild.ipynb | 47 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 01d63bc2d8d..232364b2daf 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -62,8 +62,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T15:02:36.810368900Z", - "start_time": "2023-11-10T15:02:36.799309700Z" + "end_time": "2023-11-10T16:43:15.385776400Z", + "start_time": "2023-11-10T16:43:15.342289600Z" } }, "id": "97221d43eb23768" @@ -88,8 +88,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T15:02:37.598759400Z", - "start_time": "2023-11-10T15:02:37.587165900Z" + "end_time": "2023-11-10T16:43:16.445900400Z", + "start_time": "2023-11-10T16:43:16.434349300Z" } }, "id": "68315f6ec912c58a" @@ -107,7 +107,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "outputs": [], "source": [ "config_path = 'YOUR CONFIG PATH OR ENVIRONMENT'\n", @@ -118,8 +118,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T15:02:38.968684300Z", - "start_time": "2023-11-10T15:02:38.966674300Z" + "end_time": "2023-11-10T16:44:26.046210600Z", + "start_time": "2023-11-10T16:44:26.010314700Z" } }, "id": "2505f029423b21ab" @@ -137,14 +137,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initializing usable port...\n", - "57510 ports found.\n", + "57518 ports found.\n", "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", @@ -200,7 +200,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" ] }, { @@ -272,7 +272,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" ] }, { @@ -348,8 +348,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T15:02:42.990993500Z", - "start_time": "2023-11-10T15:02:40.196779700Z" + "end_time": "2023-11-10T16:44:29.405523700Z", + "start_time": "2023-11-10T16:44:27.466895500Z" } }, "id": "7d52e3d9a1bf91cb" @@ -368,7 +368,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "outputs": [ { "name": "stdout", @@ -384,8 +384,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T15:02:45.006608600Z", - "start_time": "2023-11-10T15:02:44.999076300Z" + "end_time": "2023-11-10T16:44:33.765734100Z", + "start_time": "2023-11-10T16:44:33.763709200Z" } }, "id": "e4b88a5d482ceba4" @@ -402,15 +402,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Initializing usable port...\n", - "115020 ports found.\n", + "57518 ports found.\n", "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", @@ -464,7 +465,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" ] }, { @@ -536,7 +537,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" ] }, { @@ -569,6 +570,7 @@ "\n", "Potential Applications in Software:\n", "\n", + "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mProduct_manager\u001B[0m (to chat_manager):\n", "\n", @@ -593,6 +595,7 @@ "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\n", "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n" @@ -608,8 +611,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T15:03:27.550852700Z", - "start_time": "2023-11-10T15:03:15.297577500Z" + "end_time": "2023-11-10T16:44:39.950984400Z", + "start_time": "2023-11-10T16:44:37.978997400Z" } }, "id": "34addd498e5ab174" From 14eaf3ea376dbe8958a6b5b6454db2ffcaeff46b Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Thu, 16 Nov 2023 16:13:58 +0900 Subject: [PATCH 23/74] new feature: auto-generate agent name and system message --- ASSISTANT_CONFIG_LIST | 4 +- autobuild/agent_creator.py | 114 +++++++++++++++++++++-------- notebook/agentchat_autobuild.ipynb | 22 +++--- 3 files changed, 95 insertions(+), 45 deletions(-) diff --git a/ASSISTANT_CONFIG_LIST b/ASSISTANT_CONFIG_LIST index 1d7a5c66119..760c009ac28 100644 --- a/ASSISTANT_CONFIG_LIST +++ b/ASSISTANT_CONFIG_LIST @@ -7,7 +7,7 @@ "context": "8192", "release": "2023", "benchmark": {}, - "description": "A set of models that improve on GPT-3.5 and can understand as well as generate natural language or code." + "description": "(High cost) A set of models that improve on GPT-3.5 and can understand as well as generate natural language or code." }, "gpt-3.5-turbo": { "base": "GPT", @@ -17,7 +17,7 @@ "context": "4096", "release": "2023", "benchmark": {}, - "description": "A set of models that improve on GPT-3 and can understand as well as generate natural language or code." + "description": "(High cost) A set of models that improve on GPT-3 and can understand as well as generate natural language or code." }, "meta-llama/Llama-2-7b-chat-hf": { "base": "LLAMA", diff --git a/autobuild/agent_creator.py b/autobuild/agent_creator.py index e197fa5152f..6675cb56ce8 100644 --- a/autobuild/agent_creator.py +++ b/autobuild/agent_creator.py @@ -12,17 +12,38 @@ class AgentCreator: """ Descriptions """ - openai_server_name: str = 'openai' - max_tokens: int = 945 + openai_server_name = 'openai' + max_tokens = 945 + num_of_pos = 5 - CODING_PROMPT: str = '''Does the following task need programming - (i.e., access external API or tool by coding) to solve? + CODING_PROMPT = '''Does the following task need programming (i.e., access external API or tool by coding) to solve? TASK: {task} Answer only YES or NO. ''' + AGENT_NAME_PROMPT = '''To complete the following task, what positions/jobs should be set to maximize the efficiency? + + TASK: {task} + + Considering the effort, the position in this task should be no more then {num_of_pos}, less is better. + Answer the name of those positions/jobs, separated by comma and use "_" instead of space. + For example: Product_manager,Programmer + ''' + + AGENT_SYS_MSG_PROMPT = '''Considering the following position and corresponding task: + + TASK: {task} + POSITION: {position} + + Modify the following position requirement, let it more suitable for the above task and position: + + REQUIREMENT: {default_sys_msg} + + Your answer should omit the word "REQUIREMENT" and include the description of behavior when the task complete. + ''' + def __init__( self, host: str = 'localhost', @@ -37,9 +58,9 @@ def __init__( """ self.user_proxy: autogen.UserProxyAgent = None self.group_chat_manager_config: dict = None - self.initiate_agent_name: str = 'user' + self.initiate_agent_name: str = 'Code_interpreter' self.manager_system_message: str = 'Group chat manager.' - self.agent_configs: List[Dict] = None + self.agent_configs: List[Dict] = [] self.coding: bool = None self.default_llm_config: Dict = None self.building_task: str = None @@ -214,22 +235,53 @@ def build( ) build_manager = autogen.OpenAIWrapper(config_list=config_list) - # TODO: use the build manager to decide what agent should be created, - # and generate system message for each agent and group chat manager. if use_api: - # after this process completed, we should obtain a following list and a manager_system_config. - self.agent_configs = [ - { - 'name': 'Coder_gpt_35', - 'model': 'gpt-3.5-turbo', - 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE - }, - { - 'name': 'Product_manager', - 'model': 'gpt-3.5-turbo', - 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE - } - ] + # after this process completed, we should obtain a following list and a manager_system_message. + # self.agent_configs = [ + # { + # 'name': 'Coder_gpt_35', + # 'model': 'gpt-3.5-turbo', + # 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + # }, + # { + # 'name': 'Product_manager', + # 'model': 'gpt-3.5-turbo', + # 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + # } + # ] + resp_agent_name = build_manager.create( + messages=[ + { + "role": "user", + "content": self.AGENT_NAME_PROMPT.format(task=self.building_task, num_of_pos=self.num_of_pos) + } + ] + ).choices[0].message.content + agent_name_list = resp_agent_name.split(',') + + agent_sys_msg_list = [] + for name in agent_name_list: + resp_agent_sys_msg = build_manager.create( + messages=[ + { + "role": "user", + "content": self.AGENT_SYS_MSG_PROMPT.format( + task=self.building_task, + position=name, + default_sys_msg=autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + ) + } + ] + ).choices[0].message.content + agent_sys_msg_list.append(resp_agent_sys_msg) + + for i in range(len(agent_name_list)): + self.agent_configs.append({ + 'name': agent_name_list[i], + 'model': 'gpt-3.5-turbo', # TODO: prompt gpt-4 to select opensource model + 'system_message': agent_sys_msg_list[i] + }) + self.manager_system_message = 'Group chat manager.' for agent_config in self.agent_configs: @@ -246,10 +298,10 @@ def build( if self.coding is True: self.user_proxy = autogen.UserProxyAgent( - name="User_proxy", - system_message="A human admin.", + name="Code_interpreter", + system_message="A code interpreter interface.", code_execution_config={"last_n_messages": 2, "work_dir": "groupchat"}, - human_input_mode="TERMINATE" + human_input_mode="NEVER" ) else: self.initiate_agent_name = self.agent_configs[0]['name'] @@ -328,7 +380,7 @@ def start( self.user_proxy.initiate_chat(manager, message=task) else: for agent in agent_list: - if self.initiate_agent_name == agent.name(): + if self.initiate_agent_name == agent.name: agent.initiate_chat(manager, message=task) @@ -338,12 +390,10 @@ def start( 'temperature': 0 } task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." + building_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." - administrator = AgentCreator( - config_path=config_path - ) - administrator.build(task, default_llm_config) - administrator.save() - administrator.start(task) - administrator.clear_all_agents() + builder = AgentCreator(config_path=config_path) + builder.build(building_task, default_llm_config) + builder.start(task) + builder.clear_all_agents() diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 232364b2daf..79d31fcddb1 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -57,7 +57,7 @@ "execution_count": 1, "outputs": [], "source": [ - "task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" ], "metadata": { "collapsed": false, @@ -83,7 +83,7 @@ "execution_count": 2, "outputs": [], "source": [ - "building_task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + "building_task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" ], "metadata": { "collapsed": false, @@ -340,10 +340,10 @@ "source": [ "from autobuild import AgentCreator\n", "\n", - "administrator = AgentCreator(config_path=config_path)\n", - "administrator.build(building_task, default_llm_config)\n", - "administrator.start(task)\n", - "administrator.clear_all_agents()" + "builder = AgentCreator(config_path=config_path)\n", + "builder.build(building_task, default_llm_config)\n", + "builder.start(task)\n", + "builder.clear_all_agents()" ], "metadata": { "collapsed": false, @@ -379,7 +379,7 @@ } ], "source": [ - "saved_path = administrator.save()" + "saved_path = builder.save()" ], "metadata": { "collapsed": false, @@ -603,10 +603,10 @@ } ], "source": [ - "new_administrator = AgentCreator(config_path=config_path)\n", - "new_administrator.load(saved_path)\n", - "new_administrator.start(task)\n", - "new_administrator.clear_all_agents()" + "new_builder = AgentCreator(config_path=config_path)\n", + "new_builder.load(saved_path)\n", + "new_builder.start(task)\n", + "new_builder.clear_all_agents()" ], "metadata": { "collapsed": false, From 628ffd0f86cffb8a85b5cd880e269a71da845c63 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Mon, 20 Nov 2023 22:53:34 +0900 Subject: [PATCH 24/74] new feature: add gpts support --- autobuild/agent_creator.py | 74 +++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/autobuild/agent_creator.py b/autobuild/agent_creator.py index 6675cb56ce8..fce4d8a3f6c 100644 --- a/autobuild/agent_creator.py +++ b/autobuild/agent_creator.py @@ -6,6 +6,7 @@ import json import hashlib from typing import * +from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent class AgentCreator: @@ -16,7 +17,8 @@ class AgentCreator: max_tokens = 945 num_of_pos = 5 - CODING_PROMPT = '''Does the following task need programming (i.e., access external API or tool by coding) to solve? + CODING_PROMPT = '''Does the following task need programming (i.e., access external API or tool by coding) to solve, + or use program may help the following task become easier? TASK: {task} @@ -30,6 +32,7 @@ class AgentCreator: Considering the effort, the position in this task should be no more then {num_of_pos}, less is better. Answer the name of those positions/jobs, separated by comma and use "_" instead of space. For example: Product_manager,Programmer + Only return the list of positions. ''' AGENT_SYS_MSG_PROMPT = '''Considering the following position and corresponding task: @@ -41,38 +44,44 @@ class AgentCreator: REQUIREMENT: {default_sys_msg} - Your answer should omit the word "REQUIREMENT" and include the description of behavior when the task complete. + Hint: + # The modified requirement should not contain the code interpreter skill. + # Coding skill is limited to Python. + # Your answer should omit the word "REQUIREMENT". + # Your answer should include the description of the behavior when the task complete (user's need has been satisfied). ''' def __init__( self, host: str = 'localhost', config_path: str = 'OAI_CONFIG_LIST', + builder_model: str = 'gpt-4-1106-preview', endpoint_building_timeout: Optional[int] = 180 ): """ Args: host: endpoint host. config_path: path of the OpenAI api configs. + builder_model: specify a model as the backbone of build manager. endpoint_building_timeout: timeout for building up an endpoint server. """ + self.host = host + self.builder_model = builder_model + self.config_path = config_path + self.endpoint_building_timeout = endpoint_building_timeout + self.user_proxy: autogen.UserProxyAgent = None self.group_chat_manager_config: dict = None - self.initiate_agent_name: str = 'Code_interpreter' + self.initiate_agent_name: str = 'User_console_and_Python_code_interpreter' self.manager_system_message: str = 'Group chat manager.' self.agent_configs: List[Dict] = [] self.coding: bool = None self.default_llm_config: Dict = None self.building_task: str = None - self.open_ports: List[str] = [] self.agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} self.agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} - self.endpoint_building_timeout = endpoint_building_timeout - self.config_path = config_path - self.host = host - print('Initializing usable port...') for port in range(8000, 65535): if self._is_port_open(host, port): @@ -97,6 +106,7 @@ def create_agent( model_name_or_hf_repo: str, llm_config: dict, system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, + enable_assistant: Optional[bool] = False, world_size: Optional[int] = 1 ) -> autogen.AssistantAgent: """ @@ -110,6 +120,7 @@ def create_agent( model_name_or_hf_repo: llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). system_message: system prompt use to format an agent's behavior. + enable_assistant: use OpenAI GPTs api instead on self-construct agent. world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs). Returns: @@ -168,9 +179,20 @@ def create_agent( 'model': model_name_or_hf_repo, 'max_tokens': self.max_tokens }) - agent = autogen.AssistantAgent(name=agent_name, - llm_config=current_config.copy(), - system_message=system_message) + if enable_assistant: + # TODO: use new prompt to generate instruction. + agent = GPTAssistantAgent( + name=agent_name, + llm_config={ + **current_config, + "assistant_id": None # TODO: use previous assistant_id to reuse a previous assistant. + }, + instructions=system_message + ) + else: + agent = autogen.AssistantAgent(name=agent_name, + llm_config=current_config.copy(), + system_message=system_message) self.agent_procs_assign[agent_name] = (agent, server_id) return agent @@ -211,7 +233,8 @@ def build( building_task: str = None, default_llm_config: Dict = None, coding: bool = None, - cache_configs: Optional[Dict] = None + cache_configs: Optional[Dict] = None, + enable_assistant: Optional[bool] = False ): use_api = False @@ -230,25 +253,26 @@ def build( config_list = autogen.config_list_from_json( self.config_path, filter_dict={ - 'model': ['gpt-4'] + 'model': [self.builder_model] } ) build_manager = autogen.OpenAIWrapper(config_list=config_list) if use_api: - # after this process completed, we should obtain a following list and a manager_system_message. + # after this process completed, we should obtain a following list. # self.agent_configs = [ # { # 'name': 'Coder_gpt_35', # 'model': 'gpt-3.5-turbo', - # 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + # 'system_message': 'system message for coder' # }, # { # 'name': 'Product_manager', # 'model': 'gpt-3.5-turbo', - # 'system_message': autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE + # 'system_message': 'system message for pm' # } # ] + print('Generating agent...') resp_agent_name = build_manager.create( messages=[ { @@ -258,9 +282,11 @@ def build( ] ).choices[0].message.content agent_name_list = resp_agent_name.split(',') + print(f'{resp_agent_name} are generated.') agent_sys_msg_list = [] for name in agent_name_list: + print(f'Preparing configuration for {name}...') resp_agent_sys_msg = build_manager.create( messages=[ { @@ -278,17 +304,17 @@ def build( for i in range(len(agent_name_list)): self.agent_configs.append({ 'name': agent_name_list[i], - 'model': 'gpt-3.5-turbo', # TODO: prompt gpt-4 to select opensource model + 'model': 'gpt-4-1106-preview', # TODO: prompt gpt-4 to select opensource model 'system_message': agent_sys_msg_list[i] }) - self.manager_system_message = 'Group chat manager.' for agent_config in self.agent_configs: self.create_agent(agent_config['name'], agent_config['model'], self.default_llm_config, - system_message=agent_config['system_message']) + system_message=agent_config['system_message'], + enable_assistant=enable_assistant) if self.coding is None: resp = build_manager.create( @@ -298,8 +324,8 @@ def build( if self.coding is True: self.user_proxy = autogen.UserProxyAgent( - name="Code_interpreter", - system_message="A code interpreter interface.", + name="User_console_and_Python_code_interpreter", + system_message="User console with a python code interpreter interface.", code_execution_config={"last_n_messages": 2, "work_dir": "groupchat"}, human_input_mode="NEVER" ) @@ -321,7 +347,7 @@ def save( filepath: save path. Return: - filepath: saved path. + filepath: path save. """ if filepath is None: filepath = f'./save_config_{hashlib.md5(self.building_task.encode("utf-8")).hexdigest()}.json' @@ -376,7 +402,7 @@ def start( llm_config=self.group_chat_manager_config, system_message=self.manager_system_message) - if self.initiate_agent_name == "user" and self.user_proxy is not None: + if self.initiate_agent_name == "User_console_and_Python_code_interpreter" and self.user_proxy is not None: self.user_proxy.initiate_chat(manager, message=task) else: for agent in agent_list: @@ -393,7 +419,7 @@ def start( building_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." builder = AgentCreator(config_path=config_path) - builder.build(building_task, default_llm_config) + builder.build(building_task, default_llm_config, enable_assistant=True) builder.start(task) builder.clear_all_agents() From 863c47c778668401f6e6804e500872dc5c86357d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Tue, 21 Nov 2023 00:56:12 +0900 Subject: [PATCH 25/74] update notebook --- autobuild/__init__.py | 1 - .../agentchat/contrib/agent_builder.py | 24 +- notebook/agentchat_autobuild.ipynb | 766 ++++++++++++------ 3 files changed, 522 insertions(+), 269 deletions(-) delete mode 100644 autobuild/__init__.py rename autobuild/agent_creator.py => autogen/agentchat/contrib/agent_builder.py (96%) diff --git a/autobuild/__init__.py b/autobuild/__init__.py deleted file mode 100644 index 848d98e7a2f..00000000000 --- a/autobuild/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .agent_creator import AgentCreator \ No newline at end of file diff --git a/autobuild/agent_creator.py b/autogen/agentchat/contrib/agent_builder.py similarity index 96% rename from autobuild/agent_creator.py rename to autogen/agentchat/contrib/agent_builder.py index fce4d8a3f6c..15691335c56 100644 --- a/autobuild/agent_creator.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -9,7 +9,7 @@ from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent -class AgentCreator: +class AgentBuilder: """ Descriptions """ @@ -21,18 +21,19 @@ class AgentCreator: or use program may help the following task become easier? TASK: {task} - - Answer only YES or NO. + + Hint: + # Answer only YES or NO. ''' AGENT_NAME_PROMPT = '''To complete the following task, what positions/jobs should be set to maximize the efficiency? TASK: {task} - Considering the effort, the position in this task should be no more then {num_of_pos}, less is better. - Answer the name of those positions/jobs, separated by comma and use "_" instead of space. - For example: Product_manager,Programmer - Only return the list of positions. + Hint: + # Considering the effort, the position in this task should be no more then {num_of_pos}, less is better. + # Answer the name of those positions/jobs, separated by comma and use "_" instead of space. For example: Product_manager,Programmer + # Only return the list of positions. ''' AGENT_SYS_MSG_PROMPT = '''Considering the following position and corresponding task: @@ -180,14 +181,15 @@ def create_agent( 'max_tokens': self.max_tokens }) if enable_assistant: - # TODO: use new prompt to generate instruction. agent = GPTAssistantAgent( name=agent_name, llm_config={ **current_config, - "assistant_id": None # TODO: use previous assistant_id to reuse a previous assistant. + "assistant_id": None }, - instructions=system_message + instructions=system_message, + # TODO: use overwrite instructions to update the old instruction for an assistant. + overwrite_instructions=False ) else: agent = autogen.AssistantAgent(name=agent_name, @@ -418,7 +420,7 @@ def start( task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." building_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." - builder = AgentCreator(config_path=config_path) + builder = AgentBuilder(config_path=config_path) builder.build(building_task, default_llm_config, enable_assistant=True) builder.start(task) builder.clear_all_agents() diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 79d31fcddb1..087537ebf4b 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -3,7 +3,7 @@ { "cell_type": "markdown", "source": [ - "# Auto Builder\n", + "# AutoBuild\n", "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", @@ -62,8 +62,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T16:43:15.385776400Z", - "start_time": "2023-11-10T16:43:15.342289600Z" + "end_time": "2023-11-20T15:44:56.774895800Z", + "start_time": "2023-11-20T15:44:56.761135300Z" } }, "id": "97221d43eb23768" @@ -88,8 +88,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T16:43:16.445900400Z", - "start_time": "2023-11-10T16:43:16.434349300Z" + "end_time": "2023-11-20T15:44:56.810723300Z", + "start_time": "2023-11-20T15:44:56.764861600Z" } }, "id": "68315f6ec912c58a" @@ -107,10 +107,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "outputs": [], "source": [ - "config_path = 'YOUR CONFIG PATH OR ENVIRONMENT'\n", + "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", "default_llm_config = {\n", " 'temperature': 0\n", "}" @@ -118,8 +118,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T16:44:26.046210600Z", - "start_time": "2023-11-10T16:44:26.010314700Z" + "end_time": "2023-11-20T15:45:32.185940800Z", + "start_time": "2023-11-20T15:45:32.182940200Z" } }, "id": "2505f029423b21ab" @@ -144,55 +144,83 @@ "output_type": "stream", "text": [ "Initializing usable port...\n", - "57518 ports found.\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", + "57522 ports found.\n", + "Generating agent...\n", + "Research_analyst,Data_scientist,Software_developer are generated.\n", + "Preparing configuration for Research_analyst...\n", + "Preparing configuration for Data_scientist...\n", + "Preparing configuration for Software_developer...\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", + "\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers that mention \"GPT-4\" in their title, abstract, or full text. Once we find the paper, we can extract information about its potential applications in software from the abstract or the sections of the paper that discuss applications.\n", "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", + "Here's a Python script that uses the `requests` library to query the arXiv API for papers related to GPT-4 and then prints out the title and abstract of the most recent paper. If the abstract mentions specific applications in software, those will be included in the output.\n", "\n", - "Here is the Python code to accomplish this task:\n", + "Please note that as of my last update, GPT-4 might not have been released, and there might not be any papers specifically about it. If that's the case, the script will not return any results. However, the script is designed to work for any future papers that might be published on the topic.\n", "\n", "```python\n", "import requests\n", - "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", - "\n", - "# Parse the XML response and extract the paper details\n", - "xml_data = response.text\n", - "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", - "\n", - "# Print the paper details\n", - "print(\"Latest Paper about GPT-4 on arXiv:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", + "from datetime import datetime\n", + "\n", + "# Define the URL for the arXiv API\n", + "ARXIV_API_URL = \"http://export.arxiv.org/api/query?\"\n", + "\n", + "# Define the search parameters\n", + "search_query = \"all:GPT-4\"\n", + "start = 0\n", + "max_results = 1\n", + "sort_by = \"submittedDate\"\n", + "sort_order = \"descending\"\n", + "\n", + "# Construct the query\n", + "query = f\"search_query={search_query}&start={start}&max_results={max_results}&sortBy={sort_by}&sortOrder={sort_order}\"\n", + "\n", + "# Send the request to the arXiv API\n", + "response = requests.get(ARXIV_API_URL + query)\n", + "\n", + "# Check if the request was successful\n", + "if response.status_code == 200:\n", + " # Parse the response XML\n", + " from xml.etree import ElementTree as ET\n", + " root = ET.fromstring(response.content)\n", + "\n", + " # Find the entry element\n", + " entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", + "\n", + " if entry is not None:\n", + " # Extract the title and abstract\n", + " title = entry.find('{http://www.w3.org/2005/Atom}title').text.strip()\n", + " abstract = entry.find('{http://www.w3.org/2005/Atom}summary').text.strip()\n", + " published = entry.find('{http://www.w3.org/2005/Atom}published').text.strip()\n", + " published_date = datetime.strptime(published, '%Y-%m-%dT%H:%M:%SZ')\n", + "\n", + " # Print the title, publication date, and abstract\n", + " print(f\"Title: {title}\\nPublished Date: {published_date}\\nAbstract: {abstract}\\n\")\n", + "\n", + " # TODO: Extract potential applications in software from the abstract or full text\n", + " # This part of the task would require natural language processing and is not trivial.\n", + " # For now, we print the abstract and leave the analysis to the user.\n", + " else:\n", + " print(\"No papers found related to GPT-4 on arXiv.\")\n", + "else:\n", + " print(f\"Failed to query arXiv API. Status code: {response.status_code}\")\n", "\n", "```\n", "\n", - "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "To run this script, you will need Python installed on your system along with the `requests` library. You can install the `requests` library using pip if you don't have it already:\n", "\n", - "When you run the code, it will print the title and summary of the latest paper about GPT-4 on arXiv. It will also analyze the paper's summary and identify potential applications in software based on predefined keywords.\n", + "```bash\n", + "pip install requests\n", + "```\n", "\n", - "Let me know if you need any further assistance!\n", + "After running the script, you will need to manually analyze the abstract to determine the potential applications in software. If you need further assistance with this analysis, please let me know, and I can provide additional guidance.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[31m\n", ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" ] }, @@ -207,65 +235,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", - "exitcode: 1 (execution failed)\n", - "Code output: \n", - "Traceback (most recent call last):\n", - " File \"\", line 10, in \n", - " paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "IndexError: list index out of range\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", - "\n", - "I apologize for the error. It seems that the code failed to extract the paper details from the XML response. This could be due to changes in the XML structure or the absence of the expected tags.\n", - "\n", - "To resolve this issue, we can use the `feedparser` library to parse the XML response and extract the paper details. Here's the updated code:\n", - "\n", - "```python\n", - "import requests\n", - "import feedparser\n", - "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", - "\n", - "# Parse the XML response using feedparser\n", - "feed = feedparser.parse(response.text)\n", - "\n", - "# Extract the paper details\n", - "if len(feed.entries) > 0:\n", - " paper_title = feed.entries[0].title\n", - " paper_summary = feed.entries[0].summary\n", - "else:\n", - " print(\"No papers found for GPT-4 on arXiv.\")\n", - " exit()\n", - "\n", - "# Print the paper details\n", - "print(\"Latest Paper about GPT-4 on arXiv:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", - "```\n", - "\n", - "Please make sure you have the `requests` and `feedparser` libraries installed (`pip install requests feedparser`) before running the code.\n", - "\n", - "This updated code should be able to extract the paper details correctly. Let me know if you encounter any further issues!\n", - "\n", - "--------------------------------------------------------------------------------\n", "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is bash)...\u001B[0m\n" ] }, { @@ -279,68 +250,101 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Latest Paper about GPT-4 on arXiv:\n", - "Title: Can LLMs like GPT-4 outperform traditional AI tools in dementia\n", - " diagnosis? Maybe, but not today\n", - "Summary: Recent investigations show that large language models (LLMs), specifically\n", - "GPT-4, not only have remarkable capabilities in common Natural Language\n", - "Processing (NLP) tasks but also exhibit human-level performance on various\n", - "professional and academic benchmarks. However, whether GPT-4 can be directly\n", - "used in practical applications and replace traditional artificial intelligence\n", - "(AI) tools in specialized domains requires further experimental validation. In\n", - "this paper, we explore the potential of LLMs such as GPT-4 to outperform\n", - "traditional AI tools in dementia diagnosis. Comprehensive comparisons between\n", - "GPT-4 and traditional AI tools are conducted to examine their diagnostic\n", - "accuracy in a clinical setting. Experimental results on two real clinical\n", - "datasets show that, although LLMs like GPT-4 demonstrate potential for future\n", - "advancements in dementia diagnosis, they currently do not surpass the\n", - "performance of traditional AI tools. The interpretability and faithfulness of\n", - "GPT-4 are also evaluated by comparison with real doctors. We discuss the\n", - "limitations of GPT-4 in its current state and propose future research\n", - "directions to enhance GPT-4 in dementia diagnosis.\n", + "Title: Advancements in Generative AI: A Comprehensive Review of GANs, GPT,\n", + " Autoencoders, Diffusion Model, and Transformers\n", + "Published Date: 2023-11-17 00:08:19\n", + "Abstract: The launch of ChatGPT has garnered global attention, marking a significant\n", + "milestone in the field of Generative Artificial Intelligence. While Generative\n", + "AI has been in effect for the past decade, the introduction of ChatGPT has\n", + "ignited a new wave of research and innovation in the AI domain. This surge in\n", + "interest has led to the development and release of numerous cutting-edge tools,\n", + "such as Bard, Stable Diffusion, DALL-E, Make-A-Video, Runway ML, and Jukebox,\n", + "among others. These tools exhibit remarkable capabilities, encompassing tasks\n", + "ranging from text generation and music composition, image creation, video\n", + "production, code generation, and even scientific work. They are built upon\n", + "various state-of-the-art models, including Stable Diffusion, transformer models\n", + "like GPT-3 (recent GPT-4), variational autoencoders, and generative adversarial\n", + "networks. This advancement in Generative AI presents a wealth of exciting\n", + "opportunities and, simultaneously, unprecedented challenges. Throughout this\n", + "paper, we have explored these state-of-the-art models, the diverse array of\n", + "tasks they can accomplish, the challenges they pose, and the promising future\n", + "of Generative Artificial Intelligence.\n", + "\n", + "\n", + "Requirement already satisfied: requests in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (1.26.16)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (2023.7.22)\n", "\n", - "Potential Applications in Software:\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", + "The script has successfully found and retrieved the abstract of a recent paper titled \"Advancements in Generative AI: A Comprehensive Review of GANs, GPT, Autoencoders, Diffusion Model, and Transformers\" published on November 17, 2023. The abstract mentions the release of ChatGPT and the subsequent development of various tools that leverage state-of-the-art models, including GPT-4.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager\u001B[0m (to chat_manager):\n", + "From the abstract, we can infer potential applications of GPT-4 in software:\n", + "\n", + "1. **Text Generation**: GPT-4 can be used to generate coherent and contextually relevant text, which can be applied in chatbots, virtual assistants, and automated content creation tools.\n", + "\n", + "2. **Music Composition**: The model's generative capabilities can extend to creating music, suggesting its use in software for composing tunes or aiding musicians in the creative process.\n", "\n", - "Based on the summary of the latest paper about GPT-4 on arXiv, the potential applications in software could include:\n", + "3. **Image Creation**: GPT-4's underlying technology can be adapted for tasks like image generation, which can be used in graphic design software, game development, and virtual reality applications.\n", "\n", - "- Natural Language Processing: GPT-4 has remarkable capabilities in common NLP tasks, such as text classification, sentiment analysis, and named entity recognition.\n", - "- Text Generation: GPT-4 can generate coherent and contextually relevant text, which can be useful in various software applications, such as content generation, chatbots, and virtual assistants.\n", - "- Chatbots: GPT-4 can be used to develop more advanced and conversational chatbots that can understand and respond to user queries in a more natural and human-like manner.\n", - "- Language Translation: GPT-4's language modeling capabilities can be leveraged for language translation tasks, enabling more accurate and context-aware translations.\n", - "- Sentiment Analysis: GPT-4 can be used to analyze and understand the sentiment expressed in text, which can be valuable in applications such as social media monitoring, customer feedback analysis, and market research.\n", + "4. **Video Production**: The model could assist in generating or editing video content, which would be beneficial in the film industry, marketing, and social media content creation.\n", "\n", - "These are just a few potential applications based on the summary of the paper. The actual applications of GPT-4 in software may vary depending on specific use cases and requirements.\n", + "5. **Code Generation**: GPT-4 can potentially automate coding tasks, improve code completion tools, or assist in debugging by understanding and generating code snippets.\n", "\n", - "Let me know if there's anything else I can help you with!\n", + "6. **Scientific Work**: The model might be capable of assisting with research by generating hypotheses, writing research papers, or summarizing scientific literature.\n", + "\n", + "The paper discusses these models' capabilities, challenges, and the future of Generative AI, which likely includes further details on the applications mentioned in the abstract. For a more in-depth understanding of GPT-4's potential applications in software, one would need to read the full paper, particularly the sections that discuss GPT-4 and its applications.\n", + "\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", - "TERMINATE\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", "--------------------------------------------------------------------------------\n" ] } ], "source": [ - "from autobuild import AgentCreator\n", + "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", "\n", - "builder = AgentCreator(config_path=config_path)\n", + "builder = AgentBuilder(config_path=config_path)\n", "builder.build(building_task, default_llm_config)\n", "builder.start(task)\n", "builder.clear_all_agents()" @@ -348,8 +352,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T16:44:29.405523700Z", - "start_time": "2023-11-10T16:44:27.466895500Z" + "end_time": "2023-11-20T15:51:30.652862300Z", + "start_time": "2023-11-20T15:48:43.068765400Z" } }, "id": "7d52e3d9a1bf91cb" @@ -357,7 +361,7 @@ { "cell_type": "markdown", "source": [ - "## (Optional) Save configs\n", + "## Save configs\n", "\n", "You can save the building config by calling `save()` function. This function will save all necessary information for building all group chat agents. You can provide a specific filename, otherwise, it will save config to the current path with a generated filename 'save_config_TASK_MD5.json'." ], @@ -384,8 +388,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T16:44:33.765734100Z", - "start_time": "2023-11-10T16:44:33.763709200Z" + "end_time": "2023-11-20T15:52:01.625026700Z", + "start_time": "2023-11-20T15:52:01.560894700Z" } }, "id": "e4b88a5d482ceba4" @@ -409,55 +413,78 @@ "output_type": "stream", "text": [ "Initializing usable port...\n", - "57518 ports found.\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", - "\n", + "57522 ports found.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers related to GPT-4. Then, we can analyze the paper to identify its potential applications in software.\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers that mention \"GPT-4\" in their title, abstract, or full text. Once we find the paper, we can extract information about its potential applications in software from the abstract or the sections of the paper that discuss applications.\n", "\n", - "Here is the Python code to accomplish this task:\n", + "Here's a Python script that uses the `requests` library to query the arXiv API for papers related to GPT-4 and then prints out the title and abstract of the most recent paper. If the abstract mentions specific applications in software, those will be included in the output.\n", + "\n", + "Please note that as of my last update, GPT-4 might not have been released, and there might not be any papers specifically about it. If that's the case, the script will not return any results. However, the script is designed to work for any future papers that might be published on the topic.\n", "\n", "```python\n", "import requests\n", - "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", - "\n", - "# Parse the XML response and extract the paper details\n", - "xml_data = response.text\n", - "paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "paper_summary = xml_data.split(\"\")[1].split(\"\")[0]\n", - "\n", - "# Print the paper details\n", - "print(\"Latest Paper about GPT-4 on arXiv:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", + "from datetime import datetime\n", + "\n", + "# Define the URL for the arXiv API\n", + "ARXIV_API_URL = \"http://export.arxiv.org/api/query?\"\n", + "\n", + "# Define the search parameters\n", + "search_query = \"all:GPT-4\"\n", + "start = 0\n", + "max_results = 1\n", + "sort_by = \"submittedDate\"\n", + "sort_order = \"descending\"\n", + "\n", + "# Construct the query\n", + "query = f\"search_query={search_query}&start={start}&max_results={max_results}&sortBy={sort_by}&sortOrder={sort_order}\"\n", + "\n", + "# Send the request to the arXiv API\n", + "response = requests.get(ARXIV_API_URL + query)\n", + "\n", + "# Check if the request was successful\n", + "if response.status_code == 200:\n", + " # Parse the response XML\n", + " from xml.etree import ElementTree as ET\n", + " root = ET.fromstring(response.content)\n", + "\n", + " # Find the entry element\n", + " entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", + "\n", + " if entry is not None:\n", + " # Extract the title and abstract\n", + " title = entry.find('{http://www.w3.org/2005/Atom}title').text.strip()\n", + " abstract = entry.find('{http://www.w3.org/2005/Atom}summary').text.strip()\n", + " published = entry.find('{http://www.w3.org/2005/Atom}published').text.strip()\n", + " published_date = datetime.strptime(published, '%Y-%m-%dT%H:%M:%SZ')\n", + "\n", + " # Print the title, publication date, and abstract\n", + " print(f\"Title: {title}\\nPublished Date: {published_date}\\nAbstract: {abstract}\\n\")\n", + "\n", + " # TODO: Extract potential applications in software from the abstract or full text\n", + " # This part of the task would require natural language processing and is not trivial.\n", + " # For now, we print the abstract and leave the analysis to the user.\n", + " else:\n", + " print(\"No papers found related to GPT-4 on arXiv.\")\n", + "else:\n", + " print(f\"Failed to query arXiv API. Status code: {response.status_code}\")\n", "\n", "```\n", "\n", - "Please note that the code uses the `requests` library to make an HTTP request to the arXiv API and parse the XML response. Make sure you have the `requests` library installed (`pip install requests`) before running the code.\n", + "To run this script, you will need Python installed on your system along with the `requests` library. You can install the `requests` library using pip if you don't have it already:\n", "\n", - "When you run the code, it will print the title and summary of the latest paper about GPT-4 on arXiv. It will also analyze the paper's summary and identify potential applications in software based on predefined keywords.\n", + "```bash\n", + "pip install requests\n", + "```\n", "\n", - "Let me know if you need any further assistance!\n", + "After running the script, you will need to manually analyze the abstract to determine the potential applications in software. If you need further assistance with this analysis, please let me know, and I can provide additional guidance.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[31m\n", ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" ] }, @@ -472,64 +499,247 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is bash)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", - "exitcode: 1 (execution failed)\n", + "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Traceback (most recent call last):\n", - " File \"\", line 10, in \n", - " paper_title = xml_data.split(\"\")[2].split(\"\")[0]\n", - "IndexError: list index out of range\n", + "Title: Advancements in Generative AI: A Comprehensive Review of GANs, GPT,\n", + " Autoencoders, Diffusion Model, and Transformers\n", + "Published Date: 2023-11-17 00:08:19\n", + "Abstract: The launch of ChatGPT has garnered global attention, marking a significant\n", + "milestone in the field of Generative Artificial Intelligence. While Generative\n", + "AI has been in effect for the past decade, the introduction of ChatGPT has\n", + "ignited a new wave of research and innovation in the AI domain. This surge in\n", + "interest has led to the development and release of numerous cutting-edge tools,\n", + "such as Bard, Stable Diffusion, DALL-E, Make-A-Video, Runway ML, and Jukebox,\n", + "among others. These tools exhibit remarkable capabilities, encompassing tasks\n", + "ranging from text generation and music composition, image creation, video\n", + "production, code generation, and even scientific work. They are built upon\n", + "various state-of-the-art models, including Stable Diffusion, transformer models\n", + "like GPT-3 (recent GPT-4), variational autoencoders, and generative adversarial\n", + "networks. This advancement in Generative AI presents a wealth of exciting\n", + "opportunities and, simultaneously, unprecedented challenges. Throughout this\n", + "paper, we have explored these state-of-the-art models, the diverse array of\n", + "tasks they can accomplish, the challenges they pose, and the promising future\n", + "of Generative Artificial Intelligence.\n", + "\n", + "\n", + "Requirement already satisfied: requests in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (1.26.16)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (2023.7.22)\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", + "The script has successfully found and retrieved the abstract of a recent paper titled \"Advancements in Generative AI: A Comprehensive Review of GANs, GPT, Autoencoders, Diffusion Model, and Transformers\" published on November 17, 2023. The abstract mentions the release of ChatGPT and the subsequent development of various tools that leverage state-of-the-art models, including GPT-4.\n", + "\n", + "From the abstract, we can infer potential applications of GPT-4 in software:\n", + "\n", + "1. **Text Generation**: GPT-4 can be used to generate coherent and contextually relevant text, which can be applied in chatbots, virtual assistants, and automated content creation tools.\n", + "\n", + "2. **Music Composition**: The model's generative capabilities can extend to creating music, suggesting its use in software for composing tunes or aiding musicians in the creative process.\n", + "\n", + "3. **Image Creation**: GPT-4's underlying technology can be adapted for tasks like image generation, which can be used in graphic design software, game development, and virtual reality applications.\n", + "\n", + "4. **Video Production**: The model could assist in generating or editing video content, which would be beneficial in the film industry, marketing, and social media content creation.\n", + "\n", + "5. **Code Generation**: GPT-4 can potentially automate coding tasks, improve code completion tools, or assist in debugging by understanding and generating code snippets.\n", + "\n", + "6. **Scientific Work**: The model might be capable of assisting with research by generating hypotheses, writing research papers, or summarizing scientific literature.\n", + "\n", + "The paper discusses these models' capabilities, challenges, and the future of Generative AI, which likely includes further details on the applications mentioned in the abstract. For a more in-depth understanding of GPT-4's potential applications in software, one would need to read the full paper, particularly the sections that discuss GPT-4 and its applications.\n", + "\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", - "I apologize for the error. It seems that the code failed to extract the paper details from the XML response. This could be due to changes in the XML structure or the absence of the expected tags.\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", - "To resolve this issue, we can use the `feedparser` library to parse the XML response and extract the paper details. Here's the updated code:\n", "\n", - "```python\n", - "import requests\n", - "import feedparser\n", "\n", - "# Search for papers related to GPT-4 on arXiv\n", - "search_query = \"GPT-4\"\n", - "api_url = f\"http://export.arxiv.org/api/query?search_query=all:{search_query}&sort=submittedDate&order=descending&max_results=1\"\n", - "response = requests.get(api_url)\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "new_builder = AgentBuilder(config_path=config_path)\n", + "new_builder.load(saved_path)\n", + "new_builder.start(task)\n", + "new_builder.clear_all_agents()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-20T15:52:26.297316200Z", + "start_time": "2023-11-20T15:52:07.484284700Z" + } + }, + "id": "34addd498e5ab174" + }, + { + "cell_type": "markdown", + "source": [ + "# GPTs API support\n", + "\n", + "[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task.\n", + "AutoBuild also support GPTs api by adding `enable_assistant=True` to the `build()` function." + ], + "metadata": { + "collapsed": false + }, + "id": "32e0cf8f09eef5cd" + }, + { + "cell_type": "code", + "execution_count": 11, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Initializing usable port...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:autogen.agentchat.contrib.gpt_assistant_agent:assistant_id was None, creating a new assistant\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "57522 ports found.\n", + "Generating agent...\n", + "Research_analyst,Data_scientist,Software_developer are generated.\n", + "Preparing configuration for Research_analyst...\n", + "Preparing configuration for Data_scientist...\n", + "Preparing configuration for Software_developer...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:autogen.agentchat.contrib.gpt_assistant_agent:assistant_id was None, creating a new assistant\n", + "WARNING:autogen.agentchat.contrib.gpt_assistant_agent:assistant_id was None, creating a new assistant\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", - "# Parse the XML response using feedparser\n", - "feed = feedparser.parse(response.text)\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", + "\n", + "To locate the latest paper on GPT-4 on arXiv, I will use the arXiv API to search for articles mentioning \"GPT-4\" in their title or abstract. Once we retrieve a paper, I will attempt to extract information about its potential applications in software from the abstract or the content provided in the search result.\n", + "\n", + "Please note that the actual retrieval and processing of such specific content might vary based on the availability of the papers on arXiv and the data provided in their abstracts. Before we proceed, please be aware that accessing the content of the paper in full may require manual review, as some details might not be fully accessible via automated methods.\n", "\n", - "# Extract the paper details\n", - "if len(feed.entries) > 0:\n", - " paper_title = feed.entries[0].title\n", - " paper_summary = feed.entries[0].summary\n", + "Let's begin with the code to search for the latest paper on GPT-4 in arXiv:\n", + "\n", + "```python\n", + "import urllib.request\n", + "import urllib.parse\n", + "import json\n", + "import feedparser\n", + "\n", + "# Define a function to search arXiv for papers that match a certain query\n", + "def search_arxiv(query):\n", + " # Base URL for the arXiv API\n", + " base_url = 'http://export.arxiv.org/api/query?'\n", + "\n", + " # Construct the query parameters\n", + " params = {\n", + " 'search_query': query,\n", + " 'sortBy': 'submittedDate',\n", + " 'sortOrder': 'descending',\n", + " 'max_results': 1\n", + " }\n", + " query_string = urllib.parse.urlencode(params)\n", + "\n", + " # Make the request to arXiv API\n", + " with urllib.request.urlopen(base_url + query_string) as response:\n", + " response_text = response.read()\n", + "\n", + " # Parse the response using feedparser\n", + " feed = feedparser.parse(response_text)\n", + "\n", + " return feed.entries\n", + "\n", + "# Search for the latest paper on GPT-4\n", + "query = 'all:\"GPT-4\"'\n", + "latest_paper = search_arxiv(query)\n", + "\n", + "# Display the result\n", + "if latest_paper:\n", + " for paper in latest_paper:\n", + " print(f\"Title: {paper.title}\")\n", + " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", + " print(f\"Published on: {paper.published}\")\n", + " print(f\"Abstract: {paper.summary}\")\n", + " print(f\"Link: {paper.link}\")\n", "else:\n", - " print(\"No papers found for GPT-4 on arXiv.\")\n", - " exit()\n", - "\n", - "# Print the paper details\n", - "print(\"Latest Paper about GPT-4 on arXiv:\")\n", - "print(\"Title:\", paper_title)\n", - "print(\"Summary:\", paper_summary)\n", - "\n", - "# Analyze the potential applications in software\n", - "potential_applications = [\"Natural Language Processing\", \"Text Generation\", \"Chatbots\", \"Language Translation\", \"Sentiment Analysis\"]\n", - "print(\"\\nPotential Applications in Software:\")\n", - "for application in potential_applications:\n", - " if application.lower() in paper_summary.lower():\n", - " print(\"-\", application)\n", + " print(\"No papers on GPT-4 were found on arXiv.\")\n", + "\n", + "# After running this code, please provide the title and published date of the paper found (if any),\n", + "# and then proceed with searching the abstract for potential applications in software. \n", "```\n", "\n", - "Please make sure you have the `requests` and `feedparser` libraries installed (`pip install requests feedparser`) before running the code.\n", + "Please execute the above code. If there is a paper found, we can further delve into the abstract to identify its potential applications in software. If the abstract provides insufficient information, it may be necessary to read the full paper, which should be done by accessing the provided link manually.\n", "\n", - "This updated code should be able to extract the paper details correctly. Let me know if you encounter any further issues!\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[31m\n", ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" ] }, @@ -544,78 +754,120 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Latest Paper about GPT-4 on arXiv:\n", - "Title: Can LLMs like GPT-4 outperform traditional AI tools in dementia\n", - " diagnosis? Maybe, but not today\n", - "Summary: Recent investigations show that large language models (LLMs), specifically\n", - "GPT-4, not only have remarkable capabilities in common Natural Language\n", - "Processing (NLP) tasks but also exhibit human-level performance on various\n", - "professional and academic benchmarks. However, whether GPT-4 can be directly\n", - "used in practical applications and replace traditional artificial intelligence\n", - "(AI) tools in specialized domains requires further experimental validation. In\n", - "this paper, we explore the potential of LLMs such as GPT-4 to outperform\n", - "traditional AI tools in dementia diagnosis. Comprehensive comparisons between\n", - "GPT-4 and traditional AI tools are conducted to examine their diagnostic\n", - "accuracy in a clinical setting. Experimental results on two real clinical\n", - "datasets show that, although LLMs like GPT-4 demonstrate potential for future\n", - "advancements in dementia diagnosis, they currently do not surpass the\n", - "performance of traditional AI tools. The interpretability and faithfulness of\n", - "GPT-4 are also evaluated by comparison with real doctors. We discuss the\n", - "limitations of GPT-4 in its current state and propose future research\n", - "directions to enhance GPT-4 in dementia diagnosis.\n", + "Title: Advancements in Generative AI: A Comprehensive Review of GANs, GPT,\n", + " Autoencoders, Diffusion Model, and Transformers\n", + "Authors: Staphord Bengesi, Hoda El-Sayed, Md Kamruzzaman Sarker, Yao Houkpati, John Irungu, Timothy Oladunni\n", + "Published on: 2023-11-17T00:08:19Z\n", + "Abstract: The launch of ChatGPT has garnered global attention, marking a significant\n", + "milestone in the field of Generative Artificial Intelligence. While Generative\n", + "AI has been in effect for the past decade, the introduction of ChatGPT has\n", + "ignited a new wave of research and innovation in the AI domain. This surge in\n", + "interest has led to the development and release of numerous cutting-edge tools,\n", + "such as Bard, Stable Diffusion, DALL-E, Make-A-Video, Runway ML, and Jukebox,\n", + "among others. These tools exhibit remarkable capabilities, encompassing tasks\n", + "ranging from text generation and music composition, image creation, video\n", + "production, code generation, and even scientific work. They are built upon\n", + "various state-of-the-art models, including Stable Diffusion, transformer models\n", + "like GPT-3 (recent GPT-4), variational autoencoders, and generative adversarial\n", + "networks. This advancement in Generative AI presents a wealth of exciting\n", + "opportunities and, simultaneously, unprecedented challenges. Throughout this\n", + "paper, we have explored these state-of-the-art models, the diverse array of\n", + "tasks they can accomplish, the challenges they pose, and the promising future\n", + "of Generative Artificial Intelligence.\n", + "Link: http://arxiv.org/abs/2311.10242v1\n", "\n", - "Potential Applications in Software:\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "The paper titled \"Advancements in Generative AI: A Comprehensive Review of GANs, GPT, Autoencoders, Diffusion Model, and Transformers\" was published on November 17, 2023, by authors Staphord Bengesi, Hoda El-Sayed, Md Kamruzzaman Sarker, Yao Houkpati, John Irungu, and Timothy Oladunni. From the abstract provided, it is clear that the paper reviews the state-of-the-art models in generative artificial intelligence, including GPT-4, and their applications across various domains.\n", "\n", + "The abstract mentions that the tools developed with these models are capable of performing a broad range of tasks such as:\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mProduct_manager\u001B[0m (to chat_manager):\n", + "1. Text generation\n", + "2. Music composition\n", + "3. Image creation\n", + "4. Video production\n", + "5. Code generation\n", + "6. Scientific work\n", + "\n", + "Moreover, the paper discusses the possibilities and challenges brought forth by these advancements. It can be inferred that GPT-4, as part of this generation of AI tools, may have potential applications in the software industry that include, but are not limited to, the following:\n", "\n", - "Based on the summary of the latest paper about GPT-4 on arXiv, the potential applications in software could include:\n", + "- **Natural Language Processing (NLP)**: GPT-4 can be employed to create applications that require high levels of understanding and generation of human language, such as chatbots, virtual assistants, and customer service automation.\n", "\n", - "- Natural Language Processing: GPT-4 has remarkable capabilities in common NLP tasks, such as text classification, sentiment analysis, and named entity recognition.\n", - "- Text Generation: GPT-4 can generate coherent and contextually relevant text, which can be useful in various software applications, such as content generation, chatbots, and virtual assistants.\n", - "- Chatbots: GPT-4 can be used to develop more advanced and conversational chatbots that can understand and respond to user queries in a more natural and human-like manner.\n", - "- Language Translation: GPT-4's language modeling capabilities can be leveraged for language translation tasks, enabling more accurate and context-aware translations.\n", - "- Sentiment Analysis: GPT-4 can be used to analyze and understand the sentiment expressed in text, which can be valuable in applications such as social media monitoring, customer feedback analysis, and market research.\n", + "- **Creative Content Generation**: Given GPT-4's capability in music and image creation, it could contribute to the development of software that supports artists and creators with generating original content or enhancing their creative workflows.\n", "\n", - "These are just a few potential applications based on the summary of the paper. The actual applications of GPT-4 in software may vary depending on specific use cases and requirements.\n", + "- **Software Development Aid**: The aspect of code generation suggests that GPT-4 could be used to streamline development processes, assist in debugging, or even write code snippets, potentially integrating with development environments to enhance productivity.\n", + "\n", + "- **Film and Gaming**: In video production, GPT-4 could help with scriptwriting, storyline development, and character dialogues for movies and video games.\n", + "\n", + "- **Scientific Research**: GPT-4's use in scientific work implies potential applications in software for data analysis, hypothesis generation, and even writing research papers or summaries.\n", + "\n", + "To explore these applications in more depth, one would need to read the full paper, which can be accessed using the link provided: http://arxiv.org/abs/2311.10242v1. This task is now complete. Should you choose to delve deeper into the paper, you may gain a more detailed understanding of the specific implications and applications of GPT-4 in the software industry.\n", + "\n", + "TERMINATE\n", "\n", - "Let me know if there's anything else I can help you with!\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mUser_proxy\u001B[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "TERMINATE\n", + "\n", "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "TERMINATE\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mCoder_gpt_35\u001B[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "TERMINATE\n", + "\n", "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "TERMINATE\n", "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "TERMINATE\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "TERMINATE\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\n", + "It appears you've entered a blank response. If you have any further questions or need additional assistance, please feel free to ask. If you are finished, there's no need for further action.\n", + "\n", + "\n", "--------------------------------------------------------------------------------\n" ] } ], "source": [ - "new_builder = AgentCreator(config_path=config_path)\n", - "new_builder.load(saved_path)\n", + "new_builder = AgentBuilder(config_path=config_path)\n", + "new_builder.build(building_task, default_llm_config, enable_assistant=True) # Transfer to GPTs API.\n", "new_builder.start(task)\n", "new_builder.clear_all_agents()" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-10T16:44:39.950984400Z", - "start_time": "2023-11-10T16:44:37.978997400Z" + "end_time": "2023-11-20T15:55:01.984391700Z", + "start_time": "2023-11-20T15:52:59.180931800Z" } }, - "id": "34addd498e5ab174" + "id": "4051c25b2cd1918c" }, { "cell_type": "code", @@ -625,7 +877,7 @@ "metadata": { "collapsed": false }, - "id": "788ab82ea53d03b4" + "id": "57194293da4ae4e1" } ], "metadata": { From 7ec8952ea36e44dfbc5255b11f21537b9a753a8e Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 24 Nov 2023 03:09:52 +0900 Subject: [PATCH 26/74] update notebook --- notebook/agentchat_autobuild.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 087537ebf4b..fedbb59406f 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -83,7 +83,8 @@ "execution_count": 2, "outputs": [], "source": [ - "building_task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + "# TODO: add a different building task case.\n", + "building_task = \"SOME GENERAL DESCRIPTION, e.g., find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" ], "metadata": { "collapsed": false, @@ -110,7 +111,7 @@ "execution_count": 5, "outputs": [], "source": [ - "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST'\n", + "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # TODO: modify path\n", "default_llm_config = {\n", " 'temperature': 0\n", "}" @@ -606,8 +607,7 @@ } ], "source": [ - "new_builder = AgentBuilder(config_path=config_path)\n", - "new_builder.load(saved_path)\n", + "new_builder = AgentBuilder(config_path=config_path).load(saved_path)\n", "new_builder.start(task)\n", "new_builder.clear_all_agents()" ], @@ -856,7 +856,7 @@ ], "source": [ "new_builder = AgentBuilder(config_path=config_path)\n", - "new_builder.build(building_task, default_llm_config, enable_assistant=True) # Transfer to GPTs API.\n", + "new_builder.build(building_task, default_llm_config, enable_assistant=True) # Transfer to GPTs API. TODO: GPTs\n", "new_builder.start(task)\n", "new_builder.clear_all_agents()" ], From f7115d22182df3e53c6bf1f03f25973f20bfb89d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 24 Nov 2023 04:01:33 +0900 Subject: [PATCH 27/74] beautify; add docstring for build --- autogen/agentchat/contrib/agent_builder.py | 343 +++++++++++---------- 1 file changed, 178 insertions(+), 165 deletions(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index 15691335c56..64a8fbe6581 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -5,7 +5,7 @@ import os import json import hashlib -from typing import * +from typing import Optional, List, Dict, Tuple from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent @@ -13,51 +13,52 @@ class AgentBuilder: """ Descriptions """ - openai_server_name = 'openai' + + openai_server_name = "openai" max_tokens = 945 num_of_pos = 5 - CODING_PROMPT = '''Does the following task need programming (i.e., access external API or tool by coding) to solve, + CODING_PROMPT = """Does the following task need programming (i.e., access external API or tool by coding) to solve, or use program may help the following task become easier? TASK: {task} - + Hint: # Answer only YES or NO. - ''' + """ - AGENT_NAME_PROMPT = '''To complete the following task, what positions/jobs should be set to maximize the efficiency? + AGENT_NAME_PROMPT = """To complete the following task, what positions/jobs should be set to maximize the efficiency? TASK: {task} - + Hint: # Considering the effort, the position in this task should be no more then {num_of_pos}, less is better. # Answer the name of those positions/jobs, separated by comma and use "_" instead of space. For example: Product_manager,Programmer # Only return the list of positions. - ''' + """ + + AGENT_SYS_MSG_PROMPT = """Considering the following position and corresponding task: - AGENT_SYS_MSG_PROMPT = '''Considering the following position and corresponding task: - TASK: {task} POSITION: {position} - + Modify the following position requirement, let it more suitable for the above task and position: - + REQUIREMENT: {default_sys_msg} - + Hint: # The modified requirement should not contain the code interpreter skill. # Coding skill is limited to Python. # Your answer should omit the word "REQUIREMENT". # Your answer should include the description of the behavior when the task complete (user's need has been satisfied). - ''' + """ def __init__( - self, - host: str = 'localhost', - config_path: str = 'OAI_CONFIG_LIST', - builder_model: str = 'gpt-4-1106-preview', - endpoint_building_timeout: Optional[int] = 180 + self, + host: str = "localhost", + config_path: str = "OAI_CONFIG_LIST", + builder_model: str = "gpt-4-1106-preview", + endpoint_building_timeout: Optional[int] = 180, ): """ Args: @@ -73,8 +74,8 @@ def __init__( self.user_proxy: autogen.UserProxyAgent = None self.group_chat_manager_config: dict = None - self.initiate_agent_name: str = 'User_console_and_Python_code_interpreter' - self.manager_system_message: str = 'Group chat manager.' + self.initiate_agent_name: str = "User_console_and_Python_code_interpreter" + self.manager_system_message: str = "Group chat manager." self.agent_configs: List[Dict] = [] self.coding: bool = None self.default_llm_config: Dict = None @@ -83,11 +84,11 @@ def __init__( self.agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} self.agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} - print('Initializing usable port...') + print("Initializing usable port...") for port in range(8000, 65535): if self._is_port_open(host, port): self.open_ports.append(str(port)) - print(f'{len(self.open_ports)} ports found.') + print(f"{len(self.open_ports)} ports found.") @staticmethod def _is_port_open(host, port): @@ -102,13 +103,13 @@ def _is_port_open(host, port): return False def create_agent( - self, - agent_name: str, - model_name_or_hf_repo: str, - llm_config: dict, - system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, - enable_assistant: Optional[bool] = False, - world_size: Optional[int] = 1 + self, + agent_name: str, + model_name_or_hf_repo: str, + llm_config: dict, + system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, + enable_assistant: Optional[bool] = False, + world_size: Optional[int] = 1, ) -> autogen.AssistantAgent: """ Create a group chat agent. @@ -127,17 +128,12 @@ def create_agent( Returns: agent: a set-up agent. """ - config_list = autogen.config_list_from_json( - self.config_path, - filter_dict={ - 'model': [model_name_or_hf_repo] - } - ) - if 'gpt-' in model_name_or_hf_repo: + config_list = autogen.config_list_from_json(self.config_path, filter_dict={"model": [model_name_or_hf_repo]}) + if "gpt-" in model_name_or_hf_repo: server_id = self.openai_server_name else: - model_name = model_name_or_hf_repo.split('/')[-1] - server_id = f'{model_name}_{self.host}' + model_name = model_name_or_hf_repo.split("/")[-1] + server_id = f"{model_name}_{self.host}" if self.agent_procs.get(server_id, None) is None: while True: port = self.open_ports.pop() @@ -145,64 +141,71 @@ def create_agent( break # Use vLLM to set up a server with OpenAI API support. - agent_proc = sp.Popen(['python', '-m', 'vllm.entrypoints.openai.api_server', - f'--host', f'{self.host}', - f'--port', f'{port}', - f'--model', f'{model_name_or_hf_repo}', - f'--tensor-parallel-size', f'{world_size}'], stdout=sp.PIPE, stderr=sp.STDOUT) + agent_proc = sp.Popen( + [ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--host", + f"{self.host}", + "--port", + f"{port}", + "--model", + f"{model_name_or_hf_repo}", + "--tensor-parallel-size", + f"{world_size}", + ], + stdout=sp.PIPE, + stderr=sp.STDOUT, + ) timeout_start = time.time() while True: server_stdout = agent_proc.stdout.readline() - if server_stdout != b'': + if server_stdout != b"": print(server_stdout) timeout_end = time.time() if b"running" in server_stdout: print( - f'Running {model_name_or_hf_repo} on http://{self.host}:{port} ' - f'with tensor parallel size {world_size}.') + f"Running {model_name_or_hf_repo} on http://{self.host}:{port} " + f"with tensor parallel size {world_size}." + ) break elif b"address already in use" in server_stdout: - raise RuntimeError(f'{self.host}:{port} already in use. Fail to set up the endpoint for ' - f'{model_name_or_hf_repo} on {self.host}:{port}.') + raise RuntimeError( + f"{self.host}:{port} already in use. Fail to set up the endpoint for " + f"{model_name_or_hf_repo} on {self.host}:{port}." + ) elif timeout_end - timeout_start > self.endpoint_building_timeout: - raise RuntimeError(f'Timeout exceed. Fail to set up the endpoint for ' - f'{model_name_or_hf_repo} on {self.host}:{port}.') + raise RuntimeError( + f"Timeout exceed. Fail to set up the endpoint for " + f"{model_name_or_hf_repo} on {self.host}:{port}." + ) self.agent_procs[server_id] = (agent_proc, port) else: port = self.agent_procs[server_id][1] - config_list[0]['api_base'] = f'http://{self.host}:{port}/v1' + config_list[0]["api_base"] = f"http://{self.host}:{port}/v1" current_config = llm_config.copy() - current_config.update({ - 'config_list': config_list, - 'model': model_name_or_hf_repo, - 'max_tokens': self.max_tokens - }) + current_config.update( + {"config_list": config_list, "model": model_name_or_hf_repo, "max_tokens": self.max_tokens} + ) if enable_assistant: agent = GPTAssistantAgent( name=agent_name, - llm_config={ - **current_config, - "assistant_id": None - }, + llm_config={**current_config, "assistant_id": None}, instructions=system_message, - # TODO: use overwrite instructions to update the old instruction for an assistant. - overwrite_instructions=False + overwrite_instructions=False, ) else: - agent = autogen.AssistantAgent(name=agent_name, - llm_config=current_config.copy(), - system_message=system_message) + agent = autogen.AssistantAgent( + name=agent_name, llm_config=current_config.copy(), system_message=system_message + ) self.agent_procs_assign[agent_name] = (agent, server_id) return agent - def clear_agent( - self, - agent_name: str = None, - recycle_endpoint: bool = True - ): + def clear_agent(self, agent_name: str = None, recycle_endpoint: bool = True): """ Clear a specific agent by name. @@ -221,7 +224,7 @@ def clear_agent( if server_id == iter_sid: return self.agent_procs[server_id][0].terminate() - self.open_ports.append(server_id.split('_')[-1]) + self.open_ports.append(server_id.split("_")[-1]) def clear_all_agents(self): """ @@ -231,33 +234,38 @@ def clear_all_agents(self): self.clear_agent(agent_name) def build( - self, - building_task: str = None, - default_llm_config: Dict = None, - coding: bool = None, - cache_configs: Optional[Dict] = None, - enable_assistant: Optional[bool] = False + self, + building_task: str, + default_llm_config: Dict, + coding: bool, + cached_configs: Optional[Dict] = None, + use_gpts: Optional[bool] = False, ): + """ + Auto build agents based on the building task. + + Args: + building_task: instruction that helps build manager (gpt-4) to decide what agent should be built. + default_llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). + coding: use to identify if the user proxy (a code interpreter) should be added. + cached_configs: previously saved agent configs. + use_gpts: if true, build function will use GPTs api to build agents. + """ use_api = False - if cache_configs is None: + if cached_configs is None: use_api = True self.building_task = building_task self.default_llm_config = default_llm_config.copy() self.coding = coding else: - self.building_task = cache_configs['building_task'] - self.default_llm_config = cache_configs['default_llm_config'] - self.coding = cache_configs['coding'] - self.agent_configs = cache_configs['agent_configs'] - self.manager_system_message = cache_configs['manager_system_message'] - - config_list = autogen.config_list_from_json( - self.config_path, - filter_dict={ - 'model': [self.builder_model] - } - ) + self.building_task = cached_configs["building_task"] + self.default_llm_config = cached_configs["default_llm_config"] + self.coding = cached_configs["coding"] + self.agent_configs = cached_configs["agent_configs"] + self.manager_system_message = cached_configs["manager_system_message"] + + config_list = autogen.config_list_from_json(self.config_path, filter_dict={"model": [self.builder_model]}) build_manager = autogen.OpenAIWrapper(config_list=config_list) if use_api: @@ -274,73 +282,84 @@ def build( # 'system_message': 'system message for pm' # } # ] - print('Generating agent...') - resp_agent_name = build_manager.create( - messages=[ - { - "role": "user", - "content": self.AGENT_NAME_PROMPT.format(task=self.building_task, num_of_pos=self.num_of_pos) - } - ] - ).choices[0].message.content - agent_name_list = resp_agent_name.split(',') - print(f'{resp_agent_name} are generated.') + print("Generating agent...") + resp_agent_name = ( + build_manager.create( + messages=[ + { + "role": "user", + "content": self.AGENT_NAME_PROMPT.format( + task=self.building_task, num_of_pos=self.num_of_pos + ), + } + ] + ) + .choices[0] + .message.content + ) + agent_name_list = resp_agent_name.split(",") + print(f"{resp_agent_name} are generated.") agent_sys_msg_list = [] for name in agent_name_list: - print(f'Preparing configuration for {name}...') - resp_agent_sys_msg = build_manager.create( - messages=[ - { - "role": "user", - "content": self.AGENT_SYS_MSG_PROMPT.format( - task=self.building_task, - position=name, - default_sys_msg=autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE - ) - } - ] - ).choices[0].message.content + print(f"Preparing configuration for {name}...") + resp_agent_sys_msg = ( + build_manager.create( + messages=[ + { + "role": "user", + "content": self.AGENT_SYS_MSG_PROMPT.format( + task=self.building_task, + position=name, + default_sys_msg=autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, + ), + } + ] + ) + .choices[0] + .message.content + ) agent_sys_msg_list.append(resp_agent_sys_msg) for i in range(len(agent_name_list)): - self.agent_configs.append({ - 'name': agent_name_list[i], - 'model': 'gpt-4-1106-preview', # TODO: prompt gpt-4 to select opensource model - 'system_message': agent_sys_msg_list[i] - }) - self.manager_system_message = 'Group chat manager.' + self.agent_configs.append( + {"name": agent_name_list[i], "model": "gpt-4-1106-preview", "system_message": agent_sys_msg_list[i]} + ) + self.manager_system_message = "Group chat manager." for agent_config in self.agent_configs: - self.create_agent(agent_config['name'], - agent_config['model'], - self.default_llm_config, - system_message=agent_config['system_message'], - enable_assistant=enable_assistant) + self.create_agent( + agent_config["name"], + agent_config["model"], + self.default_llm_config, + system_message=agent_config["system_message"], + use_gpts=use_gpts, + ) if self.coding is None: - resp = build_manager.create( - messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.building_task)}] - ).choices[0].message.content - self.coding = True if resp == 'YES' else False + resp = ( + build_manager.create( + messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.building_task)}] + ) + .choices[0] + .message.content + ) + self.coding = True if resp == "YES" else False if self.coding is True: self.user_proxy = autogen.UserProxyAgent( name="User_console_and_Python_code_interpreter", system_message="User console with a python code interpreter interface.", code_execution_config={"last_n_messages": 2, "work_dir": "groupchat"}, - human_input_mode="NEVER" + human_input_mode="NEVER", ) else: - self.initiate_agent_name = self.agent_configs[0]['name'] + self.initiate_agent_name = self.agent_configs[0]["name"] self.group_chat_manager_config = self.default_llm_config.copy() - self.group_chat_manager_config['config_list'] = config_list + self.group_chat_manager_config["config_list"] = config_list - def save( - self, - filepath: Optional[str] = None - ) -> str: + def save(self, filepath: Optional[str] = None) -> str: """ Save building configs. If the filepath is not specific, this function will create a filename by encrypt the building_task string by md5 with "save_config_" prefix, and save config to the local path. @@ -353,21 +372,22 @@ def save( """ if filepath is None: filepath = f'./save_config_{hashlib.md5(self.building_task.encode("utf-8")).hexdigest()}.json' - json.dump({ - 'building_task': self.building_task, - 'agent_configs': self.agent_configs, - 'manager_system_message': self.manager_system_message, - 'coding': self.coding, - 'default_llm_config': self.default_llm_config - }, open(filepath, 'w'), indent=4) - print(f'Building config saved to {filepath}') + json.dump( + { + "building_task": self.building_task, + "agent_configs": self.agent_configs, + "manager_system_message": self.manager_system_message, + "coding": self.coding, + "default_llm_config": self.default_llm_config, + }, + open(filepath, "w"), + indent=4, + ) + print(f"Building config saved to {filepath}") return filepath - def load( - self, - filepath: str - ): + def load(self, filepath: str): """ Load building configs and call the build function to complete building without calling online LLMs' api. @@ -380,12 +400,7 @@ def load( else: raise FileNotFoundError(f"Config file {filepath} does not exist.") - def start( - self, - task: str, - max_round: Optional[int] = 12, - init_messages: Optional[List[dict]] = [] - ): + def start(self, task: str, max_round: Optional[int] = 12, init_messages: Optional[List[dict]] = []): """ Start a group chat task solving process with built config. @@ -398,11 +413,12 @@ def start( agent_list = [agent for agent, _ in self.agent_procs_assign.values()] if self.user_proxy is not None: agent_list.append(self.user_proxy) + # TODO: add agent trigger group_chat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round) - manager = autogen.GroupChatManager(groupchat=group_chat, - llm_config=self.group_chat_manager_config, - system_message=self.manager_system_message) + manager = autogen.GroupChatManager( + groupchat=group_chat, llm_config=self.group_chat_manager_config, system_message=self.manager_system_message + ) if self.initiate_agent_name == "User_console_and_Python_code_interpreter" and self.user_proxy is not None: self.user_proxy.initiate_chat(manager, message=task) @@ -412,16 +428,13 @@ def start( agent.initiate_chat(manager, message=task) -if __name__ == '__main__': - config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' - default_llm_config = { - 'temperature': 0 - } +if __name__ == "__main__": + config_path = "/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST" + default_llm_config = {"temperature": 0} task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." building_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." builder = AgentBuilder(config_path=config_path) - builder.build(building_task, default_llm_config, enable_assistant=True) + builder.build(building_task, default_llm_config, use_gpts=True) builder.start(task) builder.clear_all_agents() - From 5551e44f7b740d5ac202e37e829b018918a71cef Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sun, 26 Nov 2023 04:44:50 +0900 Subject: [PATCH 28/74] Update notebook; PR version --- autogen/agentchat/contrib/agent_builder.py | 40 +- notebook/agentchat_autobuild.ipynb | 838 +++++++++++---------- 2 files changed, 469 insertions(+), 409 deletions(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index 64a8fbe6581..d938783b578 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -11,7 +11,13 @@ class AgentBuilder: """ - Descriptions + AgentBuilder can help user build an automatic task solving process powered by multi-agent system. + Specifically, our building pipeline include init(), build(), and start(). + In build(), we prompt a gpt-4 model to create multiple participant agent and initialize a group chat, and specify + whether this task need programming to solve. + After that, user can call start() at a proper time to complete the task. + User can save the built agents' config by calling save(), and load the saved configs by load(), which can skip the + building process. """ openai_server_name = "openai" @@ -55,16 +61,16 @@ class AgentBuilder: def __init__( self, - host: str = "localhost", - config_path: str = "OAI_CONFIG_LIST", - builder_model: str = "gpt-4-1106-preview", + config_path: Optional[str] = "OAI_CONFIG_LIST", + builder_model: Optional[str] = "gpt-4-1106-preview", + host: Optional[str] = "localhost", endpoint_building_timeout: Optional[int] = 180, ): """ Args: - host: endpoint host. config_path: path of the OpenAI api configs. builder_model: specify a model as the backbone of build manager. + host: endpoint host. endpoint_building_timeout: timeout for building up an endpoint server. """ self.host = host @@ -84,11 +90,9 @@ def __init__( self.agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} self.agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} - print("Initializing usable port...") for port in range(8000, 65535): if self._is_port_open(host, port): self.open_ports.append(str(port)) - print(f"{len(self.open_ports)} ports found.") @staticmethod def _is_port_open(host, port): @@ -108,7 +112,7 @@ def create_agent( model_name_or_hf_repo: str, llm_config: dict, system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, - enable_assistant: Optional[bool] = False, + use_gpts: Optional[bool] = False, world_size: Optional[int] = 1, ) -> autogen.AssistantAgent: """ @@ -122,7 +126,7 @@ def create_agent( model_name_or_hf_repo: llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). system_message: system prompt use to format an agent's behavior. - enable_assistant: use OpenAI GPTs api instead on self-construct agent. + use_gpts: use OpenAI GPTs api instead on self-construct agent. world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs). Returns: @@ -191,7 +195,7 @@ def create_agent( current_config.update( {"config_list": config_list, "model": model_name_or_hf_repo, "max_tokens": self.max_tokens} ) - if enable_assistant: + if use_gpts: agent = GPTAssistantAgent( name=agent_name, llm_config={**current_config, "assistant_id": None}, @@ -235,9 +239,9 @@ def clear_all_agents(self): def build( self, - building_task: str, - default_llm_config: Dict, - coding: bool, + building_task: Optional[str] = None, + default_llm_config: Optional[Dict] = None, + coding: Optional[bool] = None, cached_configs: Optional[Dict] = None, use_gpts: Optional[bool] = False, ): @@ -282,7 +286,7 @@ def build( # 'system_message': 'system message for pm' # } # ] - print("Generating agent...") + print("Generating agents...") resp_agent_name = ( build_manager.create( messages=[ @@ -323,7 +327,7 @@ def build( for i in range(len(agent_name_list)): self.agent_configs.append( - {"name": agent_name_list[i], "model": "gpt-4-1106-preview", "system_message": agent_sys_msg_list[i]} + {"name": agent_name_list[i], "model": self.builder_model, "system_message": agent_sys_msg_list[i]} ) self.manager_system_message = "Group chat manager." @@ -395,11 +399,13 @@ def load(self, filepath: str): filepath: filepath for the save config. """ if os.path.isfile(filepath): - cache_configs = json.load(open(filepath)) - self.build(cache_configs=cache_configs) + cached_configs = json.load(open(filepath)) + self.build(cached_configs=cached_configs) else: raise FileNotFoundError(f"Config file {filepath} does not exist.") + return self + def start(self, task: str, max_round: Optional[int] = 12, init_messages: Optional[List[dict]] = []): """ Start a group chat task solving process with built config. diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index fedbb59406f..22eff7a8ba8 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -7,7 +7,7 @@ "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "In this notebook, we introduce a new class, `AgentCreator`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our building pipeline include `init()`, `build()`, and `start()`. In `build()`, we prompt a gpt-4 model to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. " + "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a gpt-4 model to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. " ], "metadata": { "collapsed": false @@ -19,10 +19,7 @@ "source": [ "## Requirement\n", "\n", - "Auto Agent setup need `vllm>=0.2.0` and `fastchat>=0.2.30`, you can use the following command to install the latest version of [vLLM](https://github.com/vllm-project/vllm) and [FastChat](https://github.com/lm-sys/FastChat).\n", - "```\n", - "pip install vllm \"fastchat[model_worker]\"\n", - "```" + "AutoBuild need the latest version of AutoGen. You can install AutoGen by the following command:" ], "metadata": { "collapsed": false @@ -34,7 +31,7 @@ "execution_count": null, "outputs": [], "source": [ - "!pip install vllm \"fastchat[model_worker]\"" + "!pip install pyautogen" ], "metadata": { "collapsed": false @@ -44,8 +41,8 @@ { "cell_type": "markdown", "source": [ - "## Specify a task with detailed information\n", - "Describe your task in natural language and specify your target (e.g., find some paper from arxiv)." + "## Step 1: describe your task\n", + "Describe your task and specify your target." ], "metadata": { "collapsed": false @@ -62,8 +59,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-20T15:44:56.774895800Z", - "start_time": "2023-11-20T15:44:56.761135300Z" + "end_time": "2023-11-25T19:26:52.732366100Z", + "start_time": "2023-11-25T19:26:52.708635900Z" } }, "id": "97221d43eb23768" @@ -71,7 +68,9 @@ { "cell_type": "markdown", "source": [ - "(Optional) you can specify a building task in natural language to let build manager (a LLM) build a specific group of agent for your task (e.g., \"Create a product_manager and coder for me, use gpt-3.5-turbo as backbone.\"), or just use the same task as building task and let build manager to decide which agent should be built for your task." + "## Step 2: specify a building task\n", + "\n", + "Specify a building task with a more general description. If you have only one task, you can use the task above. Building task will help build manager (a LLM) decide what agents should be build." ], "metadata": { "collapsed": false @@ -83,14 +82,13 @@ "execution_count": 2, "outputs": [], "source": [ - "# TODO: add a different building task case.\n", - "building_task = \"SOME GENERAL DESCRIPTION, e.g., find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + "building_task = \"Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-20T15:44:56.810723300Z", - "start_time": "2023-11-20T15:44:56.764861600Z" + "end_time": "2023-11-25T19:26:53.530279400Z", + "start_time": "2023-11-25T19:26:53.527267600Z" } }, "id": "68315f6ec912c58a" @@ -98,7 +96,7 @@ { "cell_type": "markdown", "source": [ - "## Prepare configuration\n", + "## Step 3: prepare configuration\n", "Prepare a `config_path` for assistant agent to limit the choice of LLM you want to use in this task. This config can be a path of json file or a name of environment variable. A `default_llm_config` is also required for initialize the specific config of LLMs like seed, temperature, etc..." ], "metadata": { @@ -108,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "outputs": [], "source": [ "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # TODO: modify path\n", @@ -119,8 +117,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-20T15:45:32.185940800Z", - "start_time": "2023-11-20T15:45:32.182940200Z" + "end_time": "2023-11-25T19:26:54.486582600Z", + "start_time": "2023-11-25T19:26:54.483568300Z" } }, "id": "2505f029423b21ab" @@ -128,8 +126,8 @@ { "cell_type": "markdown", "source": [ - "## Complete the task building\n", - "Then, we can use the `AgentCreator` to automatically complete the above task. First, initialize the AgentCreator with the predefined `task` and `config_path`. Then, call the `build` with the `default_llm_config`. GPT-4 will automatically create multiple agents and initialize a groupchat that use to complete this task. Finally, call the `start()` to start the task completion process with a groupchat. When the task begin, groupchat content between each agent will be shown in the console. This process will become interactive when the groupchat manager think that is necessary. Note that if this process involve some open-sourced LLMs (for example, LLaMA 2), an endpoint server will be automatically set up with a background process. You can call the `clear_all()` to wipe them out after the task completed." + "## Step 4: create a AgentBuilder\n", + "Create a `AgentBuilder` with the specified `config_path`. AgentBuilder will use GPT-4 in default to complete the whole process, you can also change the `builder_model` to other OpenAI model if you want." ], "metadata": { "collapsed": false @@ -138,87 +136,126 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, + "outputs": [], + "source": [ + "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", + "\n", + "builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview')" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-25T19:26:57.072116Z", + "start_time": "2023-11-25T19:26:56.147701300Z" + } + }, + "id": "bfa67c771a0fed37" + }, + { + "cell_type": "markdown", + "source": [ + "## Step 5: build group chat agents\n", + "Use `build()` to let build manager (the specified `builder_model`) complete the group chat agents generation. If you think coding is necessary in your task, you can use `coding=True` to add a user proxy (an automatic code interpreter) into the agent list. If `coding` is not specified, AutoBuilder will determine on its own whether the task requires writing code or not." + ], + "metadata": { + "collapsed": false + }, + "id": "5782dd5ecb6c217a" + }, + { + "cell_type": "code", + "execution_count": 5, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Initializing usable port...\n", - "57522 ports found.\n", - "Generating agent...\n", - "Research_analyst,Data_scientist,Software_developer are generated.\n", - "Preparing configuration for Research_analyst...\n", + "Generating agents...\n", + "Data_scientist,Research_analyst,Software_developer are generated.\n", "Preparing configuration for Data_scientist...\n", - "Preparing configuration for Software_developer...\n", + "Preparing configuration for Research_analyst...\n", + "Preparing configuration for Software_developer...\n" + ] + } + ], + "source": [ + "builder.build(building_task, default_llm_config)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-25T19:26:57.364730800Z", + "start_time": "2023-11-25T19:26:57.283681200Z" + } + }, + "id": "ab490fdbe46c0473" + }, + { + "cell_type": "markdown", + "source": [ + "## Step 6: execute task\n", + "Use agents generated in `build()` to complete the task collaboratively in a group chat." + ], + "metadata": { + "collapsed": false + }, + "id": "e00dd99880a4bf7b" + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers that mention \"GPT-4\" in their title, abstract, or full text. Once we find the paper, we can extract information about its potential applications in software from the abstract or the sections of the paper that discuss applications.\n", + "To accomplish this task, we will follow these steps:\n", "\n", - "Here's a Python script that uses the `requests` library to query the arXiv API for papers related to GPT-4 and then prints out the title and abstract of the most recent paper. If the abstract mentions specific applications in software, those will be included in the output.\n", + "1. **Search arXiv for the latest papers on GPT-4**: We will use the `arxiv` Python library to search for the most recent papers related to GPT-4. If the `arxiv` library is not installed, you will need to install it using `pip install arxiv`.\n", "\n", - "Please note that as of my last update, GPT-4 might not have been released, and there might not be any papers specifically about it. If that's the case, the script will not return any results. However, the script is designed to work for any future papers that might be published on the topic.\n", + "2. **Extract Information**: From the search results, we will extract the title, authors, summary, and publication date of the latest paper.\n", "\n", - "```python\n", - "import requests\n", - "from datetime import datetime\n", - "\n", - "# Define the URL for the arXiv API\n", - "ARXIV_API_URL = \"http://export.arxiv.org/api/query?\"\n", - "\n", - "# Define the search parameters\n", - "search_query = \"all:GPT-4\"\n", - "start = 0\n", - "max_results = 1\n", - "sort_by = \"submittedDate\"\n", - "sort_order = \"descending\"\n", - "\n", - "# Construct the query\n", - "query = f\"search_query={search_query}&start={start}&max_results={max_results}&sortBy={sort_by}&sortOrder={sort_order}\"\n", - "\n", - "# Send the request to the arXiv API\n", - "response = requests.get(ARXIV_API_URL + query)\n", - "\n", - "# Check if the request was successful\n", - "if response.status_code == 200:\n", - " # Parse the response XML\n", - " from xml.etree import ElementTree as ET\n", - " root = ET.fromstring(response.content)\n", - "\n", - " # Find the entry element\n", - " entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", - "\n", - " if entry is not None:\n", - " # Extract the title and abstract\n", - " title = entry.find('{http://www.w3.org/2005/Atom}title').text.strip()\n", - " abstract = entry.find('{http://www.w3.org/2005/Atom}summary').text.strip()\n", - " published = entry.find('{http://www.w3.org/2005/Atom}published').text.strip()\n", - " published_date = datetime.strptime(published, '%Y-%m-%dT%H:%M:%SZ')\n", - "\n", - " # Print the title, publication date, and abstract\n", - " print(f\"Title: {title}\\nPublished Date: {published_date}\\nAbstract: {abstract}\\n\")\n", - "\n", - " # TODO: Extract potential applications in software from the abstract or full text\n", - " # This part of the task would require natural language processing and is not trivial.\n", - " # For now, we print the abstract and leave the analysis to the user.\n", - " else:\n", - " print(\"No papers found related to GPT-4 on arXiv.\")\n", - "else:\n", - " print(f\"Failed to query arXiv API. Status code: {response.status_code}\")\n", + "3. **Analyze for Potential Applications in Software Development**: We will read through the summary to identify mentions of potential applications in software development.\n", "\n", - "```\n", + "4. **Present Findings**: We will print out the relevant information.\n", "\n", - "To run this script, you will need Python installed on your system along with the `requests` library. You can install the `requests` library using pip if you don't have it already:\n", + "Let's start with step 1. Here is the Python code to search for the latest papers on GPT-4 on arXiv:\n", "\n", - "```bash\n", - "pip install requests\n", + "```python\n", + "import arxiv\n", + "\n", + "# Define the search query and parameters\n", + "search_query = 'all:\"GPT-4\"'\n", + "sort_by = arxiv.SortCriterion.SubmittedDate\n", + "sort_order = arxiv.SortOrder.Descending\n", + "\n", + "# Perform the search\n", + "search = arxiv.Search(\n", + " query=search_query,\n", + " max_results=1,\n", + " sort_by=sort_by,\n", + " sort_order=sort_order\n", + ")\n", + "\n", + "# Fetch the result\n", + "for result in search.results():\n", + " print(\"Title:\", result.title)\n", + " print(\"Authors:\", \", \".join(author.name for author in result.authors))\n", + " print(\"Abstract:\", result.summary)\n", + " print(\"Publication Date:\", result.published)\n", + " print(\"URL:\", result.entry_id)\n", + " break # Since we only want the latest paper, we break after the first result\n", "```\n", "\n", - "After running the script, you will need to manually analyze the abstract to determine the potential applications in software. If you need further assistance with this analysis, please let me know, and I can provide additional guidance.\n", + "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", @@ -229,22 +266,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is bash)...\u001B[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" ] }, { @@ -255,106 +277,116 @@ "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Title: Advancements in Generative AI: A Comprehensive Review of GANs, GPT,\n", - " Autoencoders, Diffusion Model, and Transformers\n", - "Published Date: 2023-11-17 00:08:19\n", - "Abstract: The launch of ChatGPT has garnered global attention, marking a significant\n", - "milestone in the field of Generative Artificial Intelligence. While Generative\n", - "AI has been in effect for the past decade, the introduction of ChatGPT has\n", - "ignited a new wave of research and innovation in the AI domain. This surge in\n", - "interest has led to the development and release of numerous cutting-edge tools,\n", - "such as Bard, Stable Diffusion, DALL-E, Make-A-Video, Runway ML, and Jukebox,\n", - "among others. These tools exhibit remarkable capabilities, encompassing tasks\n", - "ranging from text generation and music composition, image creation, video\n", - "production, code generation, and even scientific work. They are built upon\n", - "various state-of-the-art models, including Stable Diffusion, transformer models\n", - "like GPT-3 (recent GPT-4), variational autoencoders, and generative adversarial\n", - "networks. This advancement in Generative AI presents a wealth of exciting\n", - "opportunities and, simultaneously, unprecedented challenges. Throughout this\n", - "paper, we have explored these state-of-the-art models, the diverse array of\n", - "tasks they can accomplish, the challenges they pose, and the promising future\n", - "of Generative Artificial Intelligence.\n", - "\n", - "\n", - "Requirement already satisfied: requests in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.31.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (1.26.16)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (2023.7.22)\n", + "Title: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", + "Authors: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", + "Abstract: In the field of document understanding, significant advances have been made\n", + "in the fine-tuning of Multimodal Large Language Models (MLLMs) with\n", + "instruction-following data. Nevertheless, the potential of text-grounding\n", + "capability within text-rich scenarios remains underexplored. In this paper, we\n", + "present a text-grounding document understanding model, termed TGDoc, which\n", + "addresses this deficiency by enhancing MLLMs with the ability to discern the\n", + "spatial positioning of text within images. Empirical evidence suggests that\n", + "text-grounding improves the model's interpretation of textual content, thereby\n", + "elevating its proficiency in comprehending text-rich images. Specifically, we\n", + "compile a dataset containing 99K PowerPoint presentations sourced from the\n", + "internet. We formulate instruction tuning tasks including text detection,\n", + "recognition, and spotting to facilitate the cohesive alignment between the\n", + "visual encoder and large language model. Moreover, we curate a collection of\n", + "text-rich images and prompt the text-only GPT-4 to generate 12K high-quality\n", + "conversations, featuring textual locations within text-rich scenarios. By\n", + "integrating text location data into the instructions, TGDoc is adept at\n", + "discerning text locations during the visual question process. Extensive\n", + "experiments demonstrate that our method achieves state-of-the-art performance\n", + "across multiple text-rich benchmarks, validating the effectiveness of our\n", + "method.\n", + "Publication Date: 2023-11-22 06:46:37+00:00\n", + "URL: http://arxiv.org/abs/2311.13194v1\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", - "The script has successfully found and retrieved the abstract of a recent paper titled \"Advancements in Generative AI: A Comprehensive Review of GANs, GPT, Autoencoders, Diffusion Model, and Transformers\" published on November 17, 2023. The abstract mentions the release of ChatGPT and the subsequent development of various tools that leverage state-of-the-art models, including GPT-4.\n", + "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", "\n", - "From the abstract, we can infer potential applications of GPT-4 in software:\n", + "**Abstract Summary and Potential Applications in Software Development:**\n", + "The paper discusses the enhancement of Multimodal Large Language Models (MLLMs) with text-grounding capabilities to improve document understanding. The authors present a model called TGDoc, which is designed to discern the spatial positioning of text within images. This is particularly useful for understanding text-rich images, such as PowerPoint presentations.\n", "\n", - "1. **Text Generation**: GPT-4 can be used to generate coherent and contextually relevant text, which can be applied in chatbots, virtual assistants, and automated content creation tools.\n", + "The potential applications in software development could include:\n", "\n", - "2. **Music Composition**: The model's generative capabilities can extend to creating music, suggesting its use in software for composing tunes or aiding musicians in the creative process.\n", + "1. **Document Management Systems**: The TGDoc model could be integrated into document management systems to improve the searchability and categorization of documents by understanding the spatial layout of text within images.\n", "\n", - "3. **Image Creation**: GPT-4's underlying technology can be adapted for tasks like image generation, which can be used in graphic design software, game development, and virtual reality applications.\n", + "2. **Content Creation Tools**: Software tools for creating presentations or other visual content could benefit from TGDoc by providing suggestions or automations based on the textual content within the images.\n", "\n", - "4. **Video Production**: The model could assist in generating or editing video content, which would be beneficial in the film industry, marketing, and social media content creation.\n", + "3. **Accessibility Features**: The ability to understand text within images could enhance accessibility features in software, allowing for better text-to-speech conversion for visually impaired users.\n", "\n", - "5. **Code Generation**: GPT-4 can potentially automate coding tasks, improve code completion tools, or assist in debugging by understanding and generating code snippets.\n", + "4. **Data Extraction and Analysis**: TGDoc could be used in data extraction tools to pull out relevant information from text-rich images for analysis or reporting purposes.\n", "\n", - "6. **Scientific Work**: The model might be capable of assisting with research by generating hypotheses, writing research papers, or summarizing scientific literature.\n", + "5. **Enhanced User Interfaces**: User interfaces that can interpret the spatial layout of text could provide more intuitive interactions, especially in applications that deal with document editing or creation.\n", "\n", - "The paper discusses these models' capabilities, challenges, and the future of Generative AI, which likely includes further details on the applications mentioned in the abstract. For a more in-depth understanding of GPT-4's potential applications in software, one would need to read the full paper, particularly the sections that discuss GPT-4 and its applications.\n", + "6. **Automated Customer Support**: By understanding text within images, customer support bots could provide more accurate responses to user inquiries that involve screenshots or other visual aids.\n", "\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "The paper's approach to integrating text location data into instructions for better comprehension of text-rich images could be a significant step forward in the development of more intelligent and context-aware software applications.\n", + "\n", + "**Publication Details:**\n", + "- **Title**: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", + "- **Authors**: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", + "- **Abstract URL**: [http://arxiv.org/abs/2311.13194v1](http://arxiv.org/abs/2311.13194v1)\n", + "- **Publication Date**: November 22, 2023\n", + "\n", + "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", "\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", "\n", "--------------------------------------------------------------------------------\n" ] } ], "source": [ - "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", - "\n", - "builder = AgentBuilder(config_path=config_path)\n", - "builder.build(building_task, default_llm_config)\n", - "builder.start(task)\n", - "builder.clear_all_agents()" + "builder.start(task)" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-20T15:51:30.652862300Z", - "start_time": "2023-11-20T15:48:43.068765400Z" + "end_time": "2023-11-25T19:27:43.776416600Z", + "start_time": "2023-11-25T19:27:40.439581500Z" } }, "id": "7d52e3d9a1bf91cb" @@ -362,9 +394,64 @@ { "cell_type": "markdown", "source": [ - "## Save configs\n", + "## Step 7 (Optional): clear all agents and prepare for the next task\n", + "You can clear all agents generated in this task by the following code if your task is completed or the next task is largely different from the current task." + ], + "metadata": { + "collapsed": false + }, + "id": "22a30e4b4297edd1" + }, + { + "cell_type": "code", + "execution_count": 7, + "outputs": [], + "source": [ + "builder.clear_all_agents()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-25T19:29:39.150365Z", + "start_time": "2023-11-25T19:29:39.147791300Z" + } + }, + "id": "7fb0bfff01dd1330" + }, + { + "cell_type": "markdown", + "source": [ + "## Save & load configs\n", "\n", - "You can save the building config by calling `save()` function. This function will save all necessary information for building all group chat agents. You can provide a specific filename, otherwise, it will save config to the current path with a generated filename 'save_config_TASK_MD5.json'." + "You can save all necessary information of the built group chat agents. Here is a case for those agents generated in the above task:\n", + "```json\n", + "{\n", + " \"building_task\": \"Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.\",\n", + " \"agent_configs\": [\n", + " {\n", + " \"name\": \"Data_scientist\",\n", + " \"model\": \"gpt-4-1106-preview\",\n", + " \"system_message\": \"As a Data Scientist, you are tasked with automating the retrieval and analysis of academic papers from arXiv. Utilize your Python programming acumen to develop scripts for gathering necessary information such as searching for relevant papers, downloading them, and processing their contents. Apply your analytical and language skills to interpret the data and deduce the applications of the research within specific domains.\\n\\n1. To compile information, write and implement Python scripts that search and interact with online resources, download and read files, extract content from documents, and perform other information-gathering tasks. Use the printed output as the foundation for your subsequent analysis.\\n\\n2. Execute tasks programmatically with Python scripts when possible, ensuring results are directly displayed. Approach each task with efficiency and strategic thinking.\\n\\nProgress through tasks systematically. In instances where a strategy is not provided, outline your plan before executing. Clearly distinguish between tasks handled via code and those utilizing your analytical expertise.\\n\\nWhen providing code, include only Python scripts meant to be run without user alterations. Users should execute your script as is, without modifications:\\n\\n```python\\n# filename: \\n# Python script\\nprint(\\\"Your output\\\")\\n```\\n\\nUsers should not perform any actions other than running the scripts you provide. Avoid presenting partial or incomplete scripts that require user adjustments. Refrain from requesting users to copy-paste results; instead, use the 'print' function when suitable to display outputs. Monitor the execution results they share.\\n\\nIf an error surfaces, supply corrected scripts for a re-run. If the strategy fails to resolve the issue, reassess your assumptions, gather additional details as needed, and explore alternative approaches.\\n\\nUpon successful completion of a task and verification of the results, confirm the achievement of the stated objective. Ensuring accuracy and validity of the findings is paramount. Evidence supporting your conclusions should be provided when feasible.\\n\\nUpon satisfying the user's needs and ensuring all tasks are finalized, conclude your assistance with \\\"TERMINATE\\\".\"\n", + " },\n", + " {\n", + " \"name\": \"Research_analyst\",\n", + " \"model\": \"gpt-4-1106-preview\",\n", + " \"system_message\": \"As a Research Analyst, you are expected to be a proficient AI assistant possessing a strong grasp of programming, specifically in Python, and robust analytical capabilities. Your primary responsibilities will include:\\n\\n1. Conducting comprehensive searches and retrieving information autonomously through Python scripts, such as querying databases, accessing web services (like arXiv), downloading and reading files, and retrieving system information.\\n2. Analyzing the content of the retrieved documents, particularly academic papers, and extracting insights regarding their application in specific domains, such as the potential uses of GPT-4 in software development.\\n3. Presenting your findings in a clear, detailed manner, explaining the implications of the research and its relevance to the assigned task.\\n4. Employing your programming skills to automate tasks where possible, ensuring the output is delivered through Python code with clear, executable instructions. Your code will be designed for the user to execute without amendment or additional input.\\n5. Verifying the results of information gathering and analysis to ensure accuracy and completeness, providing evidence to support your conclusions when available.\\n6. Communicating the completion of each task and confirming that the user's needs have been satisfied through a clear and conclusive statement, followed by the word \\\"TERMINATE\\\" to signal the end of the interaction.\"\n", + " },\n", + " {\n", + " \"name\": \"Software_developer\",\n", + " \"model\": \"gpt-4-1106-preview\",\n", + " \"system_message\": \"As a dedicated AI assistant for a software developer, your role involves employing your Python programming prowess and proficiency in natural language processing to facilitate the discovery and analysis of scholarly articles on arXiv. Your tasks include crafting Python scripts to automatically search, retrieve, and present information regarding the latest research, with a focus on applicable advancements in technology such as GPT-4 and its potential impact on the domain of software development.\\n\\n1. Utilize Python to programmatically seek out and extract pertinent data, for example, navigating or probing the web, downloading/ingesting documents, or showcasing content from web pages or files. When enough information has been accumulated to proceed, you will then analyze and interpret the findings.\\n\\n2. When there's a need to perform an operation programmatically, your Python code should accomplish the task and manifest the outcome. Progress through the task incrementally and systematically.\\n\\nProvide a clear plan outlining each stage of the task, specifying which components will be executed through Python coding and which through your linguistic capabilities. When proposing Python code, remember to:\\n\\n- Label the script type within the code block\\n- Avoid suggesting code that the user would need to alter\\n- Refrain from including more than one code block in your response\\n- Circumvent requesting the user to manually transcribe any results; utilize 'print' statements where applicable\\n- Examine the user's reported execution outcomes\\n\\nIf an error arises, your responsibility is to rectify the issue and submit the corrected script. Should an error remain unresolvable, or if the task remains incomplete post successful code execution, re-evaluate the scenario, gather any further required information, and formulate an alternative approach.\\n\\nUpon confirming that the task has been satisfactorily accomplished and the user's requirements have been met, indicate closure of the procedure with a concluding statement.\"\n", + " }\n", + " ],\n", + " \"manager_system_message\": \"Group chat manager.\",\n", + " \"coding\": true,\n", + " \"default_llm_config\": {\n", + " \"temperature\": 0\n", + " }\n", + "}\n", + "```\n", + "These information will be saved in JSON format. You can provide a specific filename, otherwise, AgentBuilder will save config to the current path with a generated filename 'save_config_TASK_MD5.json'." ], "metadata": { "collapsed": false @@ -379,7 +466,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Building config saved to ./save_config_5e043e06bd30f3593fdaf15e8b594002.json\n" + "Building config saved to ./save_config_eb1be857faa608aeb4c5af11fe4ab245.json\n" ] } ], @@ -389,8 +476,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-20T15:52:01.625026700Z", - "start_time": "2023-11-20T15:52:01.560894700Z" + "end_time": "2023-11-25T19:41:44.198099300Z", + "start_time": "2023-11-25T19:41:44.194085500Z" } }, "id": "e4b88a5d482ceba4" @@ -398,7 +485,7 @@ { "cell_type": "markdown", "source": [ - "Once you save the building config, you can call `load()` function with the path of the saved config to skip the building process (note that building process will call online LLMs' api)." + "After that, you can load the saved config and skip the building process. AutoBuilder will create agents with those information without prompting the builder manager." ], "metadata": { "collapsed": false @@ -413,76 +500,52 @@ "name": "stdout", "output_type": "stream", "text": [ - "Initializing usable port...\n", - "57522 ports found.\n", "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers that mention \"GPT-4\" in their title, abstract, or full text. Once we find the paper, we can extract information about its potential applications in software from the abstract or the sections of the paper that discuss applications.\n", + "To accomplish this task, we will follow these steps:\n", "\n", - "Here's a Python script that uses the `requests` library to query the arXiv API for papers related to GPT-4 and then prints out the title and abstract of the most recent paper. If the abstract mentions specific applications in software, those will be included in the output.\n", + "1. **Search arXiv for the latest papers on GPT-4**: We will use the `arxiv` Python library to search for the most recent papers related to GPT-4. If the `arxiv` library is not installed, you will need to install it using `pip install arxiv`.\n", "\n", - "Please note that as of my last update, GPT-4 might not have been released, and there might not be any papers specifically about it. If that's the case, the script will not return any results. However, the script is designed to work for any future papers that might be published on the topic.\n", + "2. **Extract Information**: From the search results, we will extract the title, authors, summary, and publication date of the latest paper.\n", "\n", - "```python\n", - "import requests\n", - "from datetime import datetime\n", - "\n", - "# Define the URL for the arXiv API\n", - "ARXIV_API_URL = \"http://export.arxiv.org/api/query?\"\n", - "\n", - "# Define the search parameters\n", - "search_query = \"all:GPT-4\"\n", - "start = 0\n", - "max_results = 1\n", - "sort_by = \"submittedDate\"\n", - "sort_order = \"descending\"\n", - "\n", - "# Construct the query\n", - "query = f\"search_query={search_query}&start={start}&max_results={max_results}&sortBy={sort_by}&sortOrder={sort_order}\"\n", - "\n", - "# Send the request to the arXiv API\n", - "response = requests.get(ARXIV_API_URL + query)\n", - "\n", - "# Check if the request was successful\n", - "if response.status_code == 200:\n", - " # Parse the response XML\n", - " from xml.etree import ElementTree as ET\n", - " root = ET.fromstring(response.content)\n", - "\n", - " # Find the entry element\n", - " entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", - "\n", - " if entry is not None:\n", - " # Extract the title and abstract\n", - " title = entry.find('{http://www.w3.org/2005/Atom}title').text.strip()\n", - " abstract = entry.find('{http://www.w3.org/2005/Atom}summary').text.strip()\n", - " published = entry.find('{http://www.w3.org/2005/Atom}published').text.strip()\n", - " published_date = datetime.strptime(published, '%Y-%m-%dT%H:%M:%SZ')\n", - "\n", - " # Print the title, publication date, and abstract\n", - " print(f\"Title: {title}\\nPublished Date: {published_date}\\nAbstract: {abstract}\\n\")\n", - "\n", - " # TODO: Extract potential applications in software from the abstract or full text\n", - " # This part of the task would require natural language processing and is not trivial.\n", - " # For now, we print the abstract and leave the analysis to the user.\n", - " else:\n", - " print(\"No papers found related to GPT-4 on arXiv.\")\n", - "else:\n", - " print(f\"Failed to query arXiv API. Status code: {response.status_code}\")\n", + "3. **Analyze for Potential Applications in Software Development**: We will read through the summary to identify mentions of potential applications in software development.\n", "\n", - "```\n", + "4. **Present Findings**: We will print out the relevant information.\n", "\n", - "To run this script, you will need Python installed on your system along with the `requests` library. You can install the `requests` library using pip if you don't have it already:\n", + "Let's start with step 1. Here is the Python code to search for the latest papers on GPT-4 on arXiv:\n", "\n", - "```bash\n", - "pip install requests\n", + "```python\n", + "import arxiv\n", + "\n", + "# Define the search query and parameters\n", + "search_query = 'all:\"GPT-4\"'\n", + "sort_by = arxiv.SortCriterion.SubmittedDate\n", + "sort_order = arxiv.SortOrder.Descending\n", + "\n", + "# Perform the search\n", + "search = arxiv.Search(\n", + " query=search_query,\n", + " max_results=1,\n", + " sort_by=sort_by,\n", + " sort_order=sort_order\n", + ")\n", + "\n", + "# Fetch the result\n", + "for result in search.results():\n", + " print(\"Title:\", result.title)\n", + " print(\"Authors:\", \", \".join(author.name for author in result.authors))\n", + " print(\"Abstract:\", result.summary)\n", + " print(\"Publication Date:\", result.published)\n", + " print(\"URL:\", result.entry_id)\n", + " break # Since we only want the latest paper, we break after the first result\n", "```\n", "\n", - "After running the script, you will need to manually analyze the abstract to determine the potential applications in software. If you need further assistance with this analysis, please let me know, and I can provide additional guidance.\n", + "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", @@ -493,22 +556,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is bash)...\u001B[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" ] }, { @@ -519,89 +567,103 @@ "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Title: Advancements in Generative AI: A Comprehensive Review of GANs, GPT,\n", - " Autoencoders, Diffusion Model, and Transformers\n", - "Published Date: 2023-11-17 00:08:19\n", - "Abstract: The launch of ChatGPT has garnered global attention, marking a significant\n", - "milestone in the field of Generative Artificial Intelligence. While Generative\n", - "AI has been in effect for the past decade, the introduction of ChatGPT has\n", - "ignited a new wave of research and innovation in the AI domain. This surge in\n", - "interest has led to the development and release of numerous cutting-edge tools,\n", - "such as Bard, Stable Diffusion, DALL-E, Make-A-Video, Runway ML, and Jukebox,\n", - "among others. These tools exhibit remarkable capabilities, encompassing tasks\n", - "ranging from text generation and music composition, image creation, video\n", - "production, code generation, and even scientific work. They are built upon\n", - "various state-of-the-art models, including Stable Diffusion, transformer models\n", - "like GPT-3 (recent GPT-4), variational autoencoders, and generative adversarial\n", - "networks. This advancement in Generative AI presents a wealth of exciting\n", - "opportunities and, simultaneously, unprecedented challenges. Throughout this\n", - "paper, we have explored these state-of-the-art models, the diverse array of\n", - "tasks they can accomplish, the challenges they pose, and the promising future\n", - "of Generative Artificial Intelligence.\n", - "\n", - "\n", - "Requirement already satisfied: requests in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.31.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (1.26.16)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (2023.7.22)\n", + "Title: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", + "Authors: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", + "Abstract: In the field of document understanding, significant advances have been made\n", + "in the fine-tuning of Multimodal Large Language Models (MLLMs) with\n", + "instruction-following data. Nevertheless, the potential of text-grounding\n", + "capability within text-rich scenarios remains underexplored. In this paper, we\n", + "present a text-grounding document understanding model, termed TGDoc, which\n", + "addresses this deficiency by enhancing MLLMs with the ability to discern the\n", + "spatial positioning of text within images. Empirical evidence suggests that\n", + "text-grounding improves the model's interpretation of textual content, thereby\n", + "elevating its proficiency in comprehending text-rich images. Specifically, we\n", + "compile a dataset containing 99K PowerPoint presentations sourced from the\n", + "internet. We formulate instruction tuning tasks including text detection,\n", + "recognition, and spotting to facilitate the cohesive alignment between the\n", + "visual encoder and large language model. Moreover, we curate a collection of\n", + "text-rich images and prompt the text-only GPT-4 to generate 12K high-quality\n", + "conversations, featuring textual locations within text-rich scenarios. By\n", + "integrating text location data into the instructions, TGDoc is adept at\n", + "discerning text locations during the visual question process. Extensive\n", + "experiments demonstrate that our method achieves state-of-the-art performance\n", + "across multiple text-rich benchmarks, validating the effectiveness of our\n", + "method.\n", + "Publication Date: 2023-11-22 06:46:37+00:00\n", + "URL: http://arxiv.org/abs/2311.13194v1\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", - "The script has successfully found and retrieved the abstract of a recent paper titled \"Advancements in Generative AI: A Comprehensive Review of GANs, GPT, Autoencoders, Diffusion Model, and Transformers\" published on November 17, 2023. The abstract mentions the release of ChatGPT and the subsequent development of various tools that leverage state-of-the-art models, including GPT-4.\n", + "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", "\n", - "From the abstract, we can infer potential applications of GPT-4 in software:\n", + "**Abstract Summary and Potential Applications in Software Development:**\n", + "The paper discusses the enhancement of Multimodal Large Language Models (MLLMs) with text-grounding capabilities to improve document understanding. The authors present a model called TGDoc, which is designed to discern the spatial positioning of text within images. This is particularly useful for understanding text-rich images, such as PowerPoint presentations.\n", "\n", - "1. **Text Generation**: GPT-4 can be used to generate coherent and contextually relevant text, which can be applied in chatbots, virtual assistants, and automated content creation tools.\n", + "The potential applications in software development could include:\n", "\n", - "2. **Music Composition**: The model's generative capabilities can extend to creating music, suggesting its use in software for composing tunes or aiding musicians in the creative process.\n", + "1. **Document Management Systems**: The TGDoc model could be integrated into document management systems to improve the searchability and categorization of documents by understanding the spatial layout of text within images.\n", "\n", - "3. **Image Creation**: GPT-4's underlying technology can be adapted for tasks like image generation, which can be used in graphic design software, game development, and virtual reality applications.\n", + "2. **Content Creation Tools**: Software tools for creating presentations or other visual content could benefit from TGDoc by providing suggestions or automations based on the textual content within the images.\n", "\n", - "4. **Video Production**: The model could assist in generating or editing video content, which would be beneficial in the film industry, marketing, and social media content creation.\n", + "3. **Accessibility Features**: The ability to understand text within images could enhance accessibility features in software, allowing for better text-to-speech conversion for visually impaired users.\n", "\n", - "5. **Code Generation**: GPT-4 can potentially automate coding tasks, improve code completion tools, or assist in debugging by understanding and generating code snippets.\n", + "4. **Data Extraction and Analysis**: TGDoc could be used in data extraction tools to pull out relevant information from text-rich images for analysis or reporting purposes.\n", "\n", - "6. **Scientific Work**: The model might be capable of assisting with research by generating hypotheses, writing research papers, or summarizing scientific literature.\n", + "5. **Enhanced User Interfaces**: User interfaces that can interpret the spatial layout of text could provide more intuitive interactions, especially in applications that deal with document editing or creation.\n", "\n", - "The paper discusses these models' capabilities, challenges, and the future of Generative AI, which likely includes further details on the applications mentioned in the abstract. For a more in-depth understanding of GPT-4's potential applications in software, one would need to read the full paper, particularly the sections that discuss GPT-4 and its applications.\n", + "6. **Automated Customer Support**: By understanding text within images, customer support bots could provide more accurate responses to user inquiries that involve screenshots or other visual aids.\n", "\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "The paper's approach to integrating text location data into instructions for better comprehension of text-rich images could be a significant step forward in the development of more intelligent and context-aware software applications.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "**Publication Details:**\n", + "- **Title**: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", + "- **Authors**: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", + "- **Abstract URL**: [http://arxiv.org/abs/2311.13194v1](http://arxiv.org/abs/2311.13194v1)\n", + "- **Publication Date**: November 22, 2023\n", + "\n", + "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software has been completed. TERMINATE.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\n", + "It seems there might be some confusion or an issue with the user console. If you have any further questions or need assistance with another task, please let me know, and I'll be happy to help. If you're done, no further action is needed. Thank you!\n", + "\n", "--------------------------------------------------------------------------------\n" ] } @@ -614,8 +676,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-20T15:52:26.297316200Z", - "start_time": "2023-11-20T15:52:07.484284700Z" + "end_time": "2023-11-25T19:41:48.044519600Z", + "start_time": "2023-11-25T19:41:45.661666200Z" } }, "id": "34addd498e5ab174" @@ -637,29 +699,21 @@ "cell_type": "code", "execution_count": 11, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Initializing usable port...\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - "WARNING:autogen.agentchat.contrib.gpt_assistant_agent:assistant_id was None, creating a new assistant\n" + "assistant_id was None, creating a new assistant\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "57522 ports found.\n", - "Generating agent...\n", - "Research_analyst,Data_scientist,Software_developer are generated.\n", - "Preparing configuration for Research_analyst...\n", + "Generating agents...\n", + "Data_scientist,Research_analyst,Software_developer are generated.\n", "Preparing configuration for Data_scientist...\n", + "Preparing configuration for Research_analyst...\n", "Preparing configuration for Software_developer...\n" ] }, @@ -667,8 +721,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:autogen.agentchat.contrib.gpt_assistant_agent:assistant_id was None, creating a new assistant\n", - "WARNING:autogen.agentchat.contrib.gpt_assistant_agent:assistant_id was None, creating a new assistant\n" + "assistant_id was None, creating a new assistant\n", + "assistant_id was None, creating a new assistant\n" ] }, { @@ -681,61 +735,72 @@ "--------------------------------------------------------------------------------\n", "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", "\n", - "To locate the latest paper on GPT-4 on arXiv, I will use the arXiv API to search for articles mentioning \"GPT-4\" in their title or abstract. Once we retrieve a paper, I will attempt to extract information about its potential applications in software from the abstract or the content provided in the search result.\n", + "To accomplish the task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software development, we will break down the process into the following steps:\n", + "\n", + "1. **Search for papers on arXiv**: We'll use the arXiv API to search for recent papers mentioning GPT-4. The arXiv API allows us to programmatically query their database.\n", "\n", - "Please note that the actual retrieval and processing of such specific content might vary based on the availability of the papers on arXiv and the data provided in their abstracts. Before we proceed, please be aware that accessing the content of the paper in full may require manual review, as some details might not be fully accessible via automated methods.\n", + "2. **Retrieve and analyze the paper**: Once we have identified relevant papers, we will download the metadata of the latest paper. We will inspect the abstract and other available metadata for mentions of potential applications in software development.\n", "\n", - "Let's begin with the code to search for the latest paper on GPT-4 in arXiv:\n", + "3. **Present findings**: Finally, we will present a summary of our findings.\n", + "\n", + "Here is a Python script that uses the arXiv API to search for the latest paper mentioning GPT-4:\n", "\n", "```python\n", + "import urllib\n", "import urllib.request\n", - "import urllib.parse\n", - "import json\n", "import feedparser\n", "\n", - "# Define a function to search arXiv for papers that match a certain query\n", + "# Define a function to search the arXiv API\n", "def search_arxiv(query):\n", " # Base URL for the arXiv API\n", " base_url = 'http://export.arxiv.org/api/query?'\n", - "\n", - " # Construct the query parameters\n", + " \n", + " # Parameters for the API query\n", " params = {\n", " 'search_query': query,\n", " 'sortBy': 'submittedDate',\n", - " 'sortOrder': 'descending',\n", - " 'max_results': 1\n", + " 'sortOrder': 'descending'\n", " }\n", + " \n", + " # Encode the parameters and make a GET request to the arXiv API\n", " query_string = urllib.parse.urlencode(params)\n", - "\n", - " # Make the request to arXiv API\n", " with urllib.request.urlopen(base_url + query_string) as response:\n", " response_text = response.read()\n", - "\n", + " \n", " # Parse the response using feedparser\n", " feed = feedparser.parse(response_text)\n", - "\n", " return feed.entries\n", "\n", - "# Search for the latest paper on GPT-4\n", - "query = 'all:\"GPT-4\"'\n", - "latest_paper = search_arxiv(query)\n", - "\n", - "# Display the result\n", - "if latest_paper:\n", - " for paper in latest_paper:\n", - " print(f\"Title: {paper.title}\")\n", - " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", - " print(f\"Published on: {paper.published}\")\n", - " print(f\"Abstract: {paper.summary}\")\n", - " print(f\"Link: {paper.link}\")\n", + "# Search for papers on GPT-4\n", + "papers = search_arxiv('all:\"GPT-4\"')\n", + "\n", + "if papers:\n", + " # Get the latest paper\n", + " latest_paper = papers[0]\n", + " \n", + " # Check for discussions of software development applications in the paper's summary\n", + " summary_lower = latest_paper.summary.lower()\n", + " software_related_terms = ['software', 'developing', 'programming', 'coding']\n", + " software_applications = any(term in summary_lower for term in software_related_terms)\n", + " \n", + " # Print out the details of the latest paper\n", + " print(f\"Title: {latest_paper.title}\")\n", + " print(f\"Authors: {', '.join(author.name for author in latest_paper.authors)}\")\n", + " print(f\"Abstract: {latest_paper.summary}\")\n", + " print(f\"Published: {latest_paper.published}\")\n", + " print(f\"Link: {latest_paper.link}\")\n", + " \n", + " # Print out whether software applications were mentioned\n", + " if software_applications:\n", + " print(\"\\nThe paper discusses potential applications in software development.\")\n", + " else:\n", + " print(\"\\nThe paper does not specifically discuss software development applications.\")\n", "else:\n", - " print(\"No papers on GPT-4 were found on arXiv.\")\n", + " print(\"No papers found on GPT-4.\")\n", "\n", - "# After running this code, please provide the title and published date of the paper found (if any),\n", - "# and then proceed with searching the abstract for potential applications in software. \n", "```\n", "\n", - "Please execute the above code. If there is a paper found, we can further delve into the abstract to identify its potential applications in software. If the abstract provides insufficient information, it may be necessary to read the full paper, which should be done by accessing the provided link manually.\n", + "Please run this script to search for and analyze the latest papers on GPT-4. The script will print out information about the most recent paper on this topic and if it finds mention of applications in software development in the abstract. If any errors occur or if the script does not produce the expected outcome, please let me know, and I will provide assistance.\n", "\n", "\n", "--------------------------------------------------------------------------------\n", @@ -747,7 +812,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:autogen.code_utils:execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" ] }, { @@ -758,86 +823,76 @@ "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Title: Advancements in Generative AI: A Comprehensive Review of GANs, GPT,\n", - " Autoencoders, Diffusion Model, and Transformers\n", - "Authors: Staphord Bengesi, Hoda El-Sayed, Md Kamruzzaman Sarker, Yao Houkpati, John Irungu, Timothy Oladunni\n", - "Published on: 2023-11-17T00:08:19Z\n", - "Abstract: The launch of ChatGPT has garnered global attention, marking a significant\n", - "milestone in the field of Generative Artificial Intelligence. While Generative\n", - "AI has been in effect for the past decade, the introduction of ChatGPT has\n", - "ignited a new wave of research and innovation in the AI domain. This surge in\n", - "interest has led to the development and release of numerous cutting-edge tools,\n", - "such as Bard, Stable Diffusion, DALL-E, Make-A-Video, Runway ML, and Jukebox,\n", - "among others. These tools exhibit remarkable capabilities, encompassing tasks\n", - "ranging from text generation and music composition, image creation, video\n", - "production, code generation, and even scientific work. They are built upon\n", - "various state-of-the-art models, including Stable Diffusion, transformer models\n", - "like GPT-3 (recent GPT-4), variational autoencoders, and generative adversarial\n", - "networks. This advancement in Generative AI presents a wealth of exciting\n", - "opportunities and, simultaneously, unprecedented challenges. Throughout this\n", - "paper, we have explored these state-of-the-art models, the diverse array of\n", - "tasks they can accomplish, the challenges they pose, and the promising future\n", - "of Generative Artificial Intelligence.\n", - "Link: http://arxiv.org/abs/2311.10242v1\n", + "Title: Towards Improving Document Understanding: An Exploration on\n", + " Text-Grounding via MLLMs\n", + "Authors: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", + "Abstract: In the field of document understanding, significant advances have been made\n", + "in the fine-tuning of Multimodal Large Language Models (MLLMs) with\n", + "instruction-following data. Nevertheless, the potential of text-grounding\n", + "capability within text-rich scenarios remains underexplored. In this paper, we\n", + "present a text-grounding document understanding model, termed TGDoc, which\n", + "addresses this deficiency by enhancing MLLMs with the ability to discern the\n", + "spatial positioning of text within images. Empirical evidence suggests that\n", + "text-grounding improves the model's interpretation of textual content, thereby\n", + "elevating its proficiency in comprehending text-rich images. Specifically, we\n", + "compile a dataset containing 99K PowerPoint presentations sourced from the\n", + "internet. We formulate instruction tuning tasks including text detection,\n", + "recognition, and spotting to facilitate the cohesive alignment between the\n", + "visual encoder and large language model. Moreover, we curate a collection of\n", + "text-rich images and prompt the text-only GPT-4 to generate 12K high-quality\n", + "conversations, featuring textual locations within text-rich scenarios. By\n", + "integrating text location data into the instructions, TGDoc is adept at\n", + "discerning text locations during the visual question process. Extensive\n", + "experiments demonstrate that our method achieves state-of-the-art performance\n", + "across multiple text-rich benchmarks, validating the effectiveness of our\n", + "method.\n", + "Published: 2023-11-22T06:46:37Z\n", + "Link: http://arxiv.org/abs/2311.13194v1\n", + "\n", + "The paper does not specifically discuss software development applications.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "The paper titled \"Advancements in Generative AI: A Comprehensive Review of GANs, GPT, Autoencoders, Diffusion Model, and Transformers\" was published on November 17, 2023, by authors Staphord Bengesi, Hoda El-Sayed, Md Kamruzzaman Sarker, Yao Houkpati, John Irungu, and Timothy Oladunni. From the abstract provided, it is clear that the paper reviews the state-of-the-art models in generative artificial intelligence, including GPT-4, and their applications across various domains.\n", - "\n", - "The abstract mentions that the tools developed with these models are capable of performing a broad range of tasks such as:\n", - "\n", - "1. Text generation\n", - "2. Music composition\n", - "3. Image creation\n", - "4. Video production\n", - "5. Code generation\n", - "6. Scientific work\n", - "\n", - "Moreover, the paper discusses the possibilities and challenges brought forth by these advancements. It can be inferred that GPT-4, as part of this generation of AI tools, may have potential applications in the software industry that include, but are not limited to, the following:\n", - "\n", - "- **Natural Language Processing (NLP)**: GPT-4 can be employed to create applications that require high levels of understanding and generation of human language, such as chatbots, virtual assistants, and customer service automation.\n", "\n", - "- **Creative Content Generation**: Given GPT-4's capability in music and image creation, it could contribute to the development of software that supports artists and creators with generating original content or enhancing their creative workflows.\n", + "It appears that the script successfully retrieved information about the latest paper related to GPT-4:\n", "\n", - "- **Software Development Aid**: The aspect of code generation suggests that GPT-4 could be used to streamline development processes, assist in debugging, or even write code snippets, potentially integrating with development environments to enhance productivity.\n", + "- **Title**: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", + "- **Authors**: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", + "- **Abstract**: The paper presents a text-grounding document understanding model, termed TGDoc, which enhances Multimodal Large Language Models (MLLMs) with the ability to discern the spatial positioning of text within images. While the paper involves text-only GPT-4 to generate high-quality conversations within text-rich scenarios, it is focused on document understanding and text-grounding, not directly on software development applications.\n", + "- **Published Date**: 2023-11-22\n", + "- **Link**: [Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs](http://arxiv.org/abs/2311.13194v1)\n", + " \n", + "Based on the abstract, the paper does not specifically discuss applications in software development. However, there might be indirect applications or implications that are not detailed in the abstract, and further reading of the full paper would be necessary to uncover those.\n", "\n", - "- **Film and Gaming**: In video production, GPT-4 could help with scriptwriting, storyline development, and character dialogues for movies and video games.\n", + "Since the provided abstract does not directly mention applications in software development, I am unable to extract insights regarding such applications without further analysis of the full document, which goes beyond the scope of this interaction.\n", "\n", - "- **Scientific Research**: GPT-4's use in scientific work implies potential applications in software for data analysis, hypothesis generation, and even writing research papers or summaries.\n", - "\n", - "To explore these applications in more depth, one would need to read the full paper, which can be accessed using the link provided: http://arxiv.org/abs/2311.10242v1. This task is now complete. Should you choose to delve deeper into the paper, you may gain a more detailed understanding of the specific implications and applications of GPT-4 in the software industry.\n", + "If needed, I can assist with the next steps or provide additional services within the constraints given.\n", "\n", "TERMINATE\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "TERMINATE\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "TERMINATE\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "TERMINATE\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "TERMINATE\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "TERMINATE\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "TERMINATE\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", @@ -845,9 +900,8 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", - "It appears you've entered a blank response. If you have any further questions or need additional assistance, please feel free to ask. If you are finished, there's no need for further action.\n", "\n", "\n", "--------------------------------------------------------------------------------\n" @@ -856,15 +910,15 @@ ], "source": [ "new_builder = AgentBuilder(config_path=config_path)\n", - "new_builder.build(building_task, default_llm_config, enable_assistant=True) # Transfer to GPTs API. TODO: GPTs\n", + "new_builder.build(building_task, default_llm_config, use_gpts=True) # Transfer to GPTs API.\n", "new_builder.start(task)\n", "new_builder.clear_all_agents()" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-20T15:55:01.984391700Z", - "start_time": "2023-11-20T15:52:59.180931800Z" + "end_time": "2023-11-25T19:43:09.367458800Z", + "start_time": "2023-11-25T19:42:15.653608Z" } }, "id": "4051c25b2cd1918c" From 6404115e8c676d62b390817b1104f1ca3794938d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sun, 26 Nov 2023 04:48:33 +0900 Subject: [PATCH 29/74] typo --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 22eff7a8ba8..ffc13df71e7 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -688,7 +688,7 @@ "# GPTs API support\n", "\n", "[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task.\n", - "AutoBuild also support GPTs api by adding `enable_assistant=True` to the `build()` function." + "AutoBuild also support GPTs api by adding `use_gpts=True` to the `build()` function." ], "metadata": { "collapsed": false From 1f537a4b91f200ed0d0009bc45736ccac4f69fd7 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sun, 26 Nov 2023 21:01:26 +0900 Subject: [PATCH 30/74] update notebook --- autogen/agentchat/contrib/agent_builder.py | 55 ++----- notebook/agentchat_autobuild.ipynb | 180 +++++++++++++-------- 2 files changed, 127 insertions(+), 108 deletions(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index d938783b578..b87897e369d 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -22,7 +22,7 @@ class AgentBuilder: openai_server_name = "openai" max_tokens = 945 - num_of_pos = 5 + max_agents = 5 # maximum number of agents build manager can create. CODING_PROMPT = """Does the following task need programming (i.e., access external API or tool by coding) to solve, or use program may help the following task become easier? @@ -38,7 +38,7 @@ class AgentBuilder: TASK: {task} Hint: - # Considering the effort, the position in this task should be no more then {num_of_pos}, less is better. + # Considering the effort, the position in this task should be no more then {max_agents}, less is better. # Answer the name of those positions/jobs, separated by comma and use "_" instead of space. For example: Product_manager,Programmer # Only return the list of positions. """ @@ -63,6 +63,7 @@ def __init__( self, config_path: Optional[str] = "OAI_CONFIG_LIST", builder_model: Optional[str] = "gpt-4-1106-preview", + agent_model: Optional[str] = "gpt-4-1106-preview", host: Optional[str] = "localhost", endpoint_building_timeout: Optional[int] = 180, ): @@ -75,6 +76,7 @@ def __init__( """ self.host = host self.builder_model = builder_model + self.agent_model = agent_model self.config_path = config_path self.endpoint_building_timeout = endpoint_building_timeout @@ -88,7 +90,7 @@ def __init__( self.building_task: str = None self.open_ports: List[str] = [] self.agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} - self.agent_procs_assign: Dict[str, Tuple[autogen.AssistantAgent, str]] = {} + self.agent_procs_assign: Dict[str, Tuple[autogen.ConversableAgent, str]] = {} for port in range(8000, 65535): if self._is_port_open(host, port): @@ -273,19 +275,6 @@ def build( build_manager = autogen.OpenAIWrapper(config_list=config_list) if use_api: - # after this process completed, we should obtain a following list. - # self.agent_configs = [ - # { - # 'name': 'Coder_gpt_35', - # 'model': 'gpt-3.5-turbo', - # 'system_message': 'system message for coder' - # }, - # { - # 'name': 'Product_manager', - # 'model': 'gpt-3.5-turbo', - # 'system_message': 'system message for pm' - # } - # ] print("Generating agents...") resp_agent_name = ( build_manager.create( @@ -293,7 +282,7 @@ def build( { "role": "user", "content": self.AGENT_NAME_PROMPT.format( - task=self.building_task, num_of_pos=self.num_of_pos + task=self.building_task, max_agents=self.max_agents ), } ] @@ -327,19 +316,18 @@ def build( for i in range(len(agent_name_list)): self.agent_configs.append( - {"name": agent_name_list[i], "model": self.builder_model, "system_message": agent_sys_msg_list[i]} + {"name": agent_name_list[i], "model": self.agent_model, "system_message": agent_sys_msg_list[i]} + ) + print(f"Creating agent {agent_name_list[i]} with backbone {self.agent_model}...") + self.create_agent( + agent_name_list[i], + self.agent_model, + self.default_llm_config, + system_message=agent_sys_msg_list[i], + use_gpts=use_gpts, ) self.manager_system_message = "Group chat manager." - for agent_config in self.agent_configs: - self.create_agent( - agent_config["name"], - agent_config["model"], - self.default_llm_config, - system_message=agent_config["system_message"], - use_gpts=use_gpts, - ) - if self.coding is None: resp = ( build_manager.create( @@ -351,6 +339,7 @@ def build( self.coding = True if resp == "YES" else False if self.coding is True: + print("Adding user console proxy...") self.user_proxy = autogen.UserProxyAgent( name="User_console_and_Python_code_interpreter", system_message="User console with a python code interpreter interface.", @@ -432,15 +421,3 @@ def start(self, task: str, max_round: Optional[int] = 12, init_messages: Optiona for agent in agent_list: if self.initiate_agent_name == agent.name: agent.initiate_chat(manager, message=task) - - -if __name__ == "__main__": - config_path = "/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST" - default_llm_config = {"temperature": 0} - task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." - building_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." - - builder = AgentBuilder(config_path=config_path) - builder.build(building_task, default_llm_config, use_gpts=True) - builder.start(task) - builder.clear_all_agents() diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index ffc13df71e7..40cb1bb9612 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -7,7 +7,7 @@ "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a gpt-4 model to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. " + "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. AgentBuilder has open-source model support, user can specify an open-source model as an agent backbone when initializing." ], "metadata": { "collapsed": false @@ -19,7 +19,8 @@ "source": [ "## Requirement\n", "\n", - "AutoBuild need the latest version of AutoGen. You can install AutoGen by the following command:" + "AutoBuild need the latest version of AutoGen.\n", + "You can install AutoGen by the following command:" ], "metadata": { "collapsed": false @@ -41,122 +42,98 @@ { "cell_type": "markdown", "source": [ - "## Step 1: describe your task\n", - "Describe your task and specify your target." + "## Step 1: prepare configuration\n", + "Prepare a `config_path` for assistant agent to limit the choice of LLM you want to use in this task. This config can be a path of json file or a name of environment variable. A `default_llm_config` is also required for initialize the specific config of LLMs like seed, temperature, etc..." ], "metadata": { "collapsed": false }, - "id": "3081403ca3b9b184" + "id": "7d0e63ab3604bdb9" }, { "cell_type": "code", "execution_count": 1, "outputs": [], "source": [ - "task = \"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-25T19:26:52.732366100Z", - "start_time": "2023-11-25T19:26:52.708635900Z" - } - }, - "id": "97221d43eb23768" - }, - { - "cell_type": "markdown", - "source": [ - "## Step 2: specify a building task\n", - "\n", - "Specify a building task with a more general description. If you have only one task, you can use the task above. Building task will help build manager (a LLM) decide what agents should be build." - ], - "metadata": { - "collapsed": false - }, - "id": "2e6a655fb6618324" - }, - { - "cell_type": "code", - "execution_count": 2, - "outputs": [], - "source": [ - "building_task = \"Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # TODO: modify path\n", + "default_llm_config = {\n", + " 'temperature': 0\n", + "}" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:26:53.530279400Z", - "start_time": "2023-11-25T19:26:53.527267600Z" + "end_time": "2023-11-26T06:42:19.833667600Z", + "start_time": "2023-11-26T06:42:19.818864300Z" } }, - "id": "68315f6ec912c58a" + "id": "2505f029423b21ab" }, { "cell_type": "markdown", "source": [ - "## Step 3: prepare configuration\n", - "Prepare a `config_path` for assistant agent to limit the choice of LLM you want to use in this task. This config can be a path of json file or a name of environment variable. A `default_llm_config` is also required for initialize the specific config of LLMs like seed, temperature, etc..." + "## Step 2: create a AgentBuilder\n", + "Create a `AgentBuilder` with the specified `config_path`. AgentBuilder will use GPT-4 in default to complete the whole process, you can also change the `builder_model` to other OpenAI model if you want." ], "metadata": { "collapsed": false }, - "id": "7d0e63ab3604bdb9" + "id": "c2d6586c68fa425b" }, { "cell_type": "code", "execution_count": 3, "outputs": [], "source": [ - "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # TODO: modify path\n", - "default_llm_config = {\n", - " 'temperature': 0\n", - "}" + "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", + "\n", + "builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview')" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:26:54.486582600Z", - "start_time": "2023-11-25T19:26:54.483568300Z" + "end_time": "2023-11-26T06:42:24.270587200Z", + "start_time": "2023-11-26T06:42:22.943534500Z" } }, - "id": "2505f029423b21ab" + "id": "bfa67c771a0fed37" }, { "cell_type": "markdown", "source": [ - "## Step 4: create a AgentBuilder\n", - "Create a `AgentBuilder` with the specified `config_path`. AgentBuilder will use GPT-4 in default to complete the whole process, you can also change the `builder_model` to other OpenAI model if you want." + "## Step 3: specify a building task\n", + "\n", + "Specify a building task with a general description. Building task will help build manager (a LLM) decide what agents should be build." ], "metadata": { "collapsed": false }, - "id": "c2d6586c68fa425b" + "id": "2e6a655fb6618324" }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "outputs": [], "source": [ - "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", - "\n", - "builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview')" + "building_task = \"Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:26:57.072116Z", - "start_time": "2023-11-25T19:26:56.147701300Z" + "start_time": "2023-11-26T06:42:20.859520700Z" } }, - "id": "bfa67c771a0fed37" + "id": "68315f6ec912c58a" }, { "cell_type": "markdown", "source": [ - "## Step 5: build group chat agents\n", - "Use `build()` to let build manager (the specified `builder_model`) complete the group chat agents generation. If you think coding is necessary in your task, you can use `coding=True` to add a user proxy (an automatic code interpreter) into the agent list. If `coding` is not specified, AutoBuilder will determine on its own whether the task requires writing code or not." + "## Step 4: build group chat agents\n", + "Use `build()` to let build manager (the specified `builder_model`) complete the group chat agents generation. If you think coding is necessary in your task, you can use `coding=True` to add a user proxy (an automatic code interpreter) into the agent list, like: \n", + "```python\n", + "builder.build(building_task, default_llm_config, coding=True)\n", + "```\n", + "If `coding` is not specified, AutoBuilder will determine on its own whether the user proxy should be added or not according to the task." ], "metadata": { "collapsed": false @@ -165,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "outputs": [ { "name": "stdout", @@ -175,7 +152,11 @@ "Data_scientist,Research_analyst,Software_developer are generated.\n", "Preparing configuration for Data_scientist...\n", "Preparing configuration for Research_analyst...\n", - "Preparing configuration for Software_developer...\n" + "Preparing configuration for Software_developer...\n", + "Creating agent Data_scientist with backbone gpt-4-1106-preview...\n", + "Creating agent Research_analyst with backbone gpt-4-1106-preview...\n", + "Creating agent Software_developer with backbone gpt-4-1106-preview...\n", + "Adding user console proxy...\n" ] } ], @@ -185,8 +166,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:26:57.364730800Z", - "start_time": "2023-11-25T19:26:57.283681200Z" + "end_time": "2023-11-26T06:42:25.995470100Z", + "start_time": "2023-11-26T06:42:25.917253700Z" } }, "id": "ab490fdbe46c0473" @@ -194,7 +175,7 @@ { "cell_type": "markdown", "source": [ - "## Step 6: execute task\n", + "## Step 5: execute task\n", "Use agents generated in `build()` to complete the task collaboratively in a group chat." ], "metadata": { @@ -380,7 +361,7 @@ } ], "source": [ - "builder.start(task)" + "builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")" ], "metadata": { "collapsed": false, @@ -394,7 +375,7 @@ { "cell_type": "markdown", "source": [ - "## Step 7 (Optional): clear all agents and prepare for the next task\n", + "## Step 6 (Optional): clear all agents and prepare for the next task\n", "You can clear all agents generated in this task by the following code if your task is completed or the next task is largely different from the current task." ], "metadata": { @@ -670,7 +651,7 @@ ], "source": [ "new_builder = AgentBuilder(config_path=config_path).load(saved_path)\n", - "new_builder.start(task)\n", + "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", "new_builder.clear_all_agents()" ], "metadata": { @@ -685,7 +666,68 @@ { "cell_type": "markdown", "source": [ - "# GPTs API support\n", + "## Open-source LLMs support\n", + "AutoBuilder support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html).\n", + "If you want to use open-source model, `vllm>=0.2.0` and `fastchat>=0.2.30` is required. You can install vLLM and fastchat by the following command:" + ], + "metadata": { + "collapsed": false + }, + "id": "f6200da25a755df" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "!pip install vllm fastchat" + ], + "metadata": { + "collapsed": false + }, + "id": "5f45061ba99c1d5" + }, + { + "cell_type": "markdown", + "source": [ + "After satisified the requirements, you can add an open-source LLM's huggingface repository to the config file, \n", + "```json\n", + "# Add the LLM's huggingface repo to your config file\n", + "[\n", + " ...\n", + " {\n", + " \"model\": \"meta-llama/Llama-2-7b-chat\",\n", + " \"api_key\": \"EMPTY\"\n", + " }\n", + "]\n", + "```\n", + "and specify it when initializing AgentBuilder. \n", + "AgentBuilder will automatically set up an endpoint server for open-source LLM, make sure you have suffcient GPUs resources." + ], + "metadata": { + "collapsed": false + }, + "id": "f7af6180af22bec9" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "new_builder = AgentBuilder(config_path=config_path, agent_model='meta-llama/Llama-2-7b-chat') # use llama-2-7b as agent backbone.\n", + "new_builder.build(building_task, default_llm_config)\n", + "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", + "new_builder.clear_all_agents()" + ], + "metadata": { + "collapsed": false + }, + "id": "4b16fb43a4dc60f1" + }, + { + "cell_type": "markdown", + "source": [ + "## GPTs API support\n", "\n", "[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task.\n", "AutoBuild also support GPTs api by adding `use_gpts=True` to the `build()` function." @@ -911,7 +953,7 @@ "source": [ "new_builder = AgentBuilder(config_path=config_path)\n", "new_builder.build(building_task, default_llm_config, use_gpts=True) # Transfer to GPTs API.\n", - "new_builder.start(task)\n", + "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", "new_builder.clear_all_agents()" ], "metadata": { From 1c6fb7cd86df1b0be05ffaf91970de5ec2998db0 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sun, 26 Nov 2023 21:07:12 +0900 Subject: [PATCH 31/74] fix typo --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 40cb1bb9612..b6988c2541c 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -692,7 +692,7 @@ "source": [ "After satisified the requirements, you can add an open-source LLM's huggingface repository to the config file, \n", "```json\n", - "# Add the LLM's huggingface repo to your config file\n", + "// Add the LLM's huggingface repo to your config file\n", "[\n", " ...\n", " {\n", From be8a203bc14016fbcdcd34a610265ac505bd7e89 Mon Sep 17 00:00:00 2001 From: JieyuZ2 Date: Sun, 26 Nov 2023 14:03:14 -0800 Subject: [PATCH 32/74] try to run llama --- autogen/agentchat/contrib/agent_builder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index b87897e369d..26b5f3ff26b 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -246,6 +246,7 @@ def build( coding: Optional[bool] = None, cached_configs: Optional[Dict] = None, use_gpts: Optional[bool] = False, + **kwargs, ): """ Auto build agents based on the building task. @@ -325,6 +326,7 @@ def build( self.default_llm_config, system_message=agent_sys_msg_list[i], use_gpts=use_gpts, + **kwargs, ) self.manager_system_message = "Group chat manager." From d999ee08bb7cd0f42ab309983191ed2e34f1252c Mon Sep 17 00:00:00 2001 From: JieyuZ2 Date: Sun, 26 Nov 2023 14:03:27 -0800 Subject: [PATCH 33/74] try to run llama --- notebook/agentchat_autobuild.ipynb | 549 ++++++++++++++++++----------- 1 file changed, 340 insertions(+), 209 deletions(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index b6988c2541c..62d234a9782 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -2,131 +2,165 @@ "cells": [ { "cell_type": "markdown", + "id": "c1004af6a7fbfcd8", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "# AutoBuild\n", "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. AgentBuilder has open-source model support, user can specify an open-source model as an agent backbone when initializing." - ], - "metadata": { - "collapsed": false - }, - "id": "c1004af6a7fbfcd8" + ] }, { "cell_type": "markdown", + "id": "ec78dda8e3826d8a", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "## Requirement\n", "\n", "AutoBuild need the latest version of AutoGen.\n", "You can install AutoGen by the following command:" - ], - "metadata": { - "collapsed": false - }, - "id": "ec78dda8e3826d8a" + ] }, { "cell_type": "code", "execution_count": null, + "id": "e8e9ae50658be975", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [], "source": [ "!pip install pyautogen" - ], - "metadata": { - "collapsed": false - }, - "id": "e8e9ae50658be975" + ] }, { "cell_type": "markdown", + "id": "7d0e63ab3604bdb9", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "## Step 1: prepare configuration\n", "Prepare a `config_path` for assistant agent to limit the choice of LLM you want to use in this task. This config can be a path of json file or a name of environment variable. A `default_llm_config` is also required for initialize the specific config of LLMs like seed, temperature, etc..." - ], - "metadata": { - "collapsed": false - }, - "id": "7d0e63ab3604bdb9" + ] }, { "cell_type": "code", - "execution_count": 1, - "outputs": [], - "source": [ - "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # TODO: modify path\n", - "default_llm_config = {\n", - " 'temperature': 0\n", - "}" - ], + "execution_count": 2, + "id": "2505f029423b21ab", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-26T06:42:19.833667600Z", "start_time": "2023-11-26T06:42:19.818864300Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false } }, - "id": "2505f029423b21ab" + "outputs": [], + "source": [ + "config_path = '/home/murphy/data-ssd/dynabench/autogen/OAI_CONFIG_LIST_sample' # TODO: modify path\n", + "default_llm_config = {\n", + " 'temperature': 0\n", + "}" + ] }, { "cell_type": "markdown", + "id": "c2d6586c68fa425b", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "## Step 2: create a AgentBuilder\n", "Create a `AgentBuilder` with the specified `config_path`. AgentBuilder will use GPT-4 in default to complete the whole process, you can also change the `builder_model` to other OpenAI model if you want." - ], - "metadata": { - "collapsed": false - }, - "id": "c2d6586c68fa425b" + ] }, { "cell_type": "code", "execution_count": 3, - "outputs": [], - "source": [ - "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", - "\n", - "builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview')" - ], + "id": "bfa67c771a0fed37", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-26T06:42:24.270587200Z", "start_time": "2023-11-26T06:42:22.943534500Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false } }, - "id": "bfa67c771a0fed37" + "outputs": [], + "source": [ + "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", + "\n", + "builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview')" + ] }, { "cell_type": "markdown", + "id": "2e6a655fb6618324", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "## Step 3: specify a building task\n", "\n", "Specify a building task with a general description. Building task will help build manager (a LLM) decide what agents should be build." - ], - "metadata": { - "collapsed": false - }, - "id": "2e6a655fb6618324" + ] }, { "cell_type": "code", - "execution_count": 2, - "outputs": [], - "source": [ - "building_task = \"Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" - ], + "execution_count": 4, + "id": "68315f6ec912c58a", "metadata": { - "collapsed": false, "ExecuteTime": { "start_time": "2023-11-26T06:42:20.859520700Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false } }, - "id": "68315f6ec912c58a" + "outputs": [], + "source": [ + "building_task = \"Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + ] }, { "cell_type": "markdown", + "id": "5782dd5ecb6c217a", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "## Step 4: build group chat agents\n", "Use `build()` to let build manager (the specified `builder_model`) complete the group chat agents generation. If you think coding is necessary in your task, you can use `coding=True` to add a user proxy (an automatic code interpreter) into the agent list, like: \n", @@ -134,15 +168,22 @@ "builder.build(building_task, default_llm_config, coding=True)\n", "```\n", "If `coding` is not specified, AutoBuilder will determine on its own whether the user proxy should be added or not according to the task." - ], - "metadata": { - "collapsed": false - }, - "id": "5782dd5ecb6c217a" + ] }, { "cell_type": "code", "execution_count": 4, + "id": "ab490fdbe46c0473", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-26T06:42:25.995470100Z", + "start_time": "2023-11-26T06:42:25.917253700Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [ { "name": "stdout", @@ -162,41 +203,47 @@ ], "source": [ "builder.build(building_task, default_llm_config)" - ], + ] + }, + { + "cell_type": "markdown", + "id": "e00dd99880a4bf7b", "metadata": { "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-26T06:42:25.995470100Z", - "start_time": "2023-11-26T06:42:25.917253700Z" + "jupyter": { + "outputs_hidden": false } }, - "id": "ab490fdbe46c0473" - }, - { - "cell_type": "markdown", "source": [ "## Step 5: execute task\n", "Use agents generated in `build()` to complete the task collaboratively in a group chat." - ], - "metadata": { - "collapsed": false - }, - "id": "e00dd99880a4bf7b" + ] }, { "cell_type": "code", "execution_count": 6, + "id": "7d52e3d9a1bf91cb", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-25T19:27:43.776416600Z", + "start_time": "2023-11-25T19:27:40.439581500Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", + "\u001b[33mSoftware_developer\u001b[0m (to chat_manager):\n", "\n", "To accomplish this task, we will follow these steps:\n", "\n", @@ -239,8 +286,8 @@ "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" ] }, { @@ -254,7 +301,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -285,7 +332,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", "\n", "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", "\n", @@ -317,42 +364,42 @@ "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", @@ -362,45 +409,50 @@ ], "source": [ "builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")" - ], + ] + }, + { + "cell_type": "markdown", + "id": "22a30e4b4297edd1", "metadata": { "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-25T19:27:43.776416600Z", - "start_time": "2023-11-25T19:27:40.439581500Z" + "jupyter": { + "outputs_hidden": false } }, - "id": "7d52e3d9a1bf91cb" - }, - { - "cell_type": "markdown", "source": [ "## Step 6 (Optional): clear all agents and prepare for the next task\n", "You can clear all agents generated in this task by the following code if your task is completed or the next task is largely different from the current task." - ], - "metadata": { - "collapsed": false - }, - "id": "22a30e4b4297edd1" + ] }, { "cell_type": "code", "execution_count": 7, - "outputs": [], - "source": [ - "builder.clear_all_agents()" - ], + "id": "7fb0bfff01dd1330", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-25T19:29:39.150365Z", "start_time": "2023-11-25T19:29:39.147791300Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false } }, - "id": "7fb0bfff01dd1330" + "outputs": [], + "source": [ + "builder.clear_all_agents()" + ] }, { "cell_type": "markdown", + "id": "bbb098638a086898", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "## Save & load configs\n", "\n", @@ -433,15 +485,22 @@ "}\n", "```\n", "These information will be saved in JSON format. You can provide a specific filename, otherwise, AgentBuilder will save config to the current path with a generated filename 'save_config_TASK_MD5.json'." - ], - "metadata": { - "collapsed": false - }, - "id": "bbb098638a086898" + ] }, { "cell_type": "code", "execution_count": 8, + "id": "e4b88a5d482ceba4", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-25T19:41:44.198099300Z", + "start_time": "2023-11-25T19:41:44.194085500Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [ { "name": "stdout", @@ -453,40 +512,46 @@ ], "source": [ "saved_path = builder.save()" - ], + ] + }, + { + "cell_type": "markdown", + "id": "a35620c10ee42be", "metadata": { "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-25T19:41:44.198099300Z", - "start_time": "2023-11-25T19:41:44.194085500Z" + "jupyter": { + "outputs_hidden": false } }, - "id": "e4b88a5d482ceba4" - }, - { - "cell_type": "markdown", "source": [ "After that, you can load the saved config and skip the building process. AutoBuilder will create agents with those information without prompting the builder manager." - ], - "metadata": { - "collapsed": false - }, - "id": "a35620c10ee42be" + ] }, { "cell_type": "code", "execution_count": 9, + "id": "34addd498e5ab174", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-25T19:41:48.044519600Z", + "start_time": "2023-11-25T19:41:45.661666200Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", + "\u001b[33mSoftware_developer\u001b[0m (to chat_manager):\n", "\n", "To accomplish this task, we will follow these steps:\n", "\n", @@ -529,8 +594,8 @@ "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" ] }, { @@ -544,7 +609,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -575,7 +640,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", "\n", "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", "\n", @@ -607,41 +672,41 @@ "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", "\n", "It seems there might be some confusion or an issue with the user console. If you have any further questions or need assistance with another task, please let me know, and I'll be happy to help. If you're done, no further action is needed. Thank you!\n", "\n", @@ -653,93 +718,164 @@ "new_builder = AgentBuilder(config_path=config_path).load(saved_path)\n", "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", "new_builder.clear_all_agents()" - ], + ] + }, + { + "cell_type": "markdown", + "id": "f6200da25a755df", "metadata": { "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-25T19:41:48.044519600Z", - "start_time": "2023-11-25T19:41:45.661666200Z" + "jupyter": { + "outputs_hidden": false } }, - "id": "34addd498e5ab174" - }, - { - "cell_type": "markdown", "source": [ "## Open-source LLMs support\n", "AutoBuilder support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html).\n", "If you want to use open-source model, `vllm>=0.2.0` and `fastchat>=0.2.30` is required. You can install vLLM and fastchat by the following command:" - ], - "metadata": { - "collapsed": false - }, - "id": "f6200da25a755df" + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [ - "!pip install vllm fastchat" - ], + "id": "5f45061ba99c1d5", "metadata": { - "collapsed": false + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } }, - "id": "5f45061ba99c1d5" + "outputs": [], + "source": [ + "!pip install vllm\n", + "!pip install \"fschat[model_worker,webui]\"" + ] }, { "cell_type": "markdown", + "id": "f7af6180af22bec9", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ - "After satisified the requirements, you can add an open-source LLM's huggingface repository to the config file, \n", + "After satisfying the requirements, you can add an open-source LLM's huggingface repository to the config file,\n", "```json\n", "// Add the LLM's huggingface repo to your config file\n", "[\n", " ...\n", " {\n", - " \"model\": \"meta-llama/Llama-2-7b-chat\",\n", + " \"model\": \"meta-llama/Llama-2-13b-chat-hf\",\n", " \"api_key\": \"EMPTY\"\n", " }\n", "]\n", "```\n", "and specify it when initializing AgentBuilder. \n", - "AgentBuilder will automatically set up an endpoint server for open-source LLM, make sure you have suffcient GPUs resources." - ], - "metadata": { - "collapsed": false - }, - "id": "f7af6180af22bec9" + "AgentBuilder will automatically set up an endpoint server for open-source LLM, make sure you have sufficient GPUs resources." + ] }, { "cell_type": "code", - "execution_count": null, - "outputs": [], + "execution_count": 5, + "id": "4b16fb43a4dc60f1", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generating agents...\n", + "Data_scientist,Research_analyst,Software_developer are generated.\n", + "Preparing configuration for Data_scientist...\n", + "Preparing configuration for Research_analyst...\n", + "Preparing configuration for Software_developer...\n", + "Creating agent Data_scientist with backbone meta-llama/Llama-2-7b-chat-hf...\n", + "b\"INFO 11-26 14:01:39 llm_engine.py:72] Initializing an LLM engine with config: model='meta-llama/Llama-2-7b-chat-hf', tokenizer='meta-llama/Llama-2-7b-chat-hf', tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, quantization=None, seed=0)\\n\"\n", + "b\"INFO 11-26 14:01:39 tokenizer.py:31] For some LLaMA V1 models, initializing the fast tokenizer may take a long time. To reduce the initialization time, consider using 'hf-internal-testing/llama-tokenizer' instead of the original tokenizer.\\n\"\n", + "b'INFO 11-26 14:01:45 llm_engine.py:207] # GPU blocks: 3803, # CPU blocks: 512\\n'\n", + "b\"INFO 11-26 14:01:46 tokenizer.py:31] For some LLaMA V1 models, initializing the fast tokenizer may take a long time. To reduce the initialization time, consider using 'hf-internal-testing/llama-tokenizer' instead of the original tokenizer.\\n\"\n", + "b'INFO: Started server process [340198]\\n'\n", + "b'INFO: Waiting for application startup.\\n'\n", + "b'INFO: Application startup complete.\\n'\n", + "b'INFO: Uvicorn running on http://localhost:65534 (Press CTRL+C to quit)\\n'\n", + "Running meta-llama/Llama-2-7b-chat-hf on http://localhost:65534 with tensor parallel size 1.\n", + "Creating agent Research_analyst with backbone meta-llama/Llama-2-7b-chat-hf...\n", + "Creating agent Software_developer with backbone meta-llama/Llama-2-7b-chat-hf...\n", + "Adding user console proxy...\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "ename": "TypeError", + "evalue": "Completions.create() got an unexpected keyword argument 'api_base'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[5], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m new_builder \u001b[38;5;241m=\u001b[39m AgentBuilder(config_path\u001b[38;5;241m=\u001b[39mconfig_path, agent_model\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmeta-llama/Llama-2-7b-chat-hf\u001b[39m\u001b[38;5;124m'\u001b[39m, endpoint_building_timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1000000000000000000000\u001b[39m) \u001b[38;5;66;03m# use llama-2-13b-chat as agent backbone.\u001b[39;00m\n\u001b[1;32m 2\u001b[0m new_builder\u001b[38;5;241m.\u001b[39mbuild(building_task, default_llm_config)\n\u001b[0;32m----> 3\u001b[0m \u001b[43mnew_builder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m new_builder\u001b[38;5;241m.\u001b[39mclear_all_agents()\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/contrib/agent_builder.py:421\u001b[0m, in \u001b[0;36mAgentBuilder.start\u001b[0;34m(self, task, max_round, init_messages)\u001b[0m\n\u001b[1;32m 416\u001b[0m manager \u001b[38;5;241m=\u001b[39m autogen\u001b[38;5;241m.\u001b[39mGroupChatManager(\n\u001b[1;32m 417\u001b[0m groupchat\u001b[38;5;241m=\u001b[39mgroup_chat, llm_config\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroup_chat_manager_config, system_message\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmanager_system_message\n\u001b[1;32m 418\u001b[0m )\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minitiate_agent_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUser_console_and_Python_code_interpreter\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muser_proxy \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muser_proxy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minitiate_chat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmanager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessage\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 422\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 423\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent \u001b[38;5;129;01min\u001b[39;00m agent_list:\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:550\u001b[0m, in \u001b[0;36mConversableAgent.initiate_chat\u001b[0;34m(self, recipient, clear_history, silent, **context)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Initiate a chat with the recipient agent.\u001b[39;00m\n\u001b[1;32m 537\u001b[0m \n\u001b[1;32m 538\u001b[0m \u001b[38;5;124;03mReset the consecutive auto reply counter.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 547\u001b[0m \u001b[38;5;124;03m \"message\" needs to be provided if the `generate_init_message` method is not overridden.\u001b[39;00m\n\u001b[1;32m 548\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 549\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_chat(recipient, clear_history)\n\u001b[0;32m--> 550\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_init_message\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrecipient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msilent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msilent\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:348\u001b[0m, in \u001b[0;36mConversableAgent.send\u001b[0;34m(self, message, recipient, request_reply, silent)\u001b[0m\n\u001b[1;32m 346\u001b[0m valid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_append_oai_message(message, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massistant\u001b[39m\u001b[38;5;124m\"\u001b[39m, recipient)\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m valid:\n\u001b[0;32m--> 348\u001b[0m \u001b[43mrecipient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreceive\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest_reply\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msilent\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 349\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 350\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 351\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMessage can\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt be converted into a valid ChatCompletion message. Either content or function_call must be provided.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 352\u001b[0m )\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:481\u001b[0m, in \u001b[0;36mConversableAgent.receive\u001b[0;34m(self, message, sender, request_reply, silent)\u001b[0m\n\u001b[1;32m 479\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m request_reply \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m request_reply \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreply_at_receive[sender] \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m:\n\u001b[1;32m 480\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[0;32m--> 481\u001b[0m reply \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_reply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchat_messages\u001b[49m\u001b[43m[\u001b[49m\u001b[43msender\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msender\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 482\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m reply \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 483\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msend(reply, sender, silent\u001b[38;5;241m=\u001b[39msilent)\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:906\u001b[0m, in \u001b[0;36mConversableAgent.generate_reply\u001b[0;34m(self, messages, sender, exclude)\u001b[0m\n\u001b[1;32m 904\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m 905\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_match_trigger(reply_func_tuple[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrigger\u001b[39m\u001b[38;5;124m\"\u001b[39m], sender):\n\u001b[0;32m--> 906\u001b[0m final, reply \u001b[38;5;241m=\u001b[39m \u001b[43mreply_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msender\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreply_func_tuple\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mconfig\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 907\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m final:\n\u001b[1;32m 908\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m reply\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/groupchat.py:271\u001b[0m, in \u001b[0;36mGroupChatManager.run_chat\u001b[0;34m(self, messages, sender, config)\u001b[0m\n\u001b[1;32m 269\u001b[0m speaker \u001b[38;5;241m=\u001b[39m groupchat\u001b[38;5;241m.\u001b[39mselect_speaker(speaker, \u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m 270\u001b[0m \u001b[38;5;66;03m# let the speaker speak\u001b[39;00m\n\u001b[0;32m--> 271\u001b[0m reply \u001b[38;5;241m=\u001b[39m \u001b[43mspeaker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_reply\u001b[49m\u001b[43m(\u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 272\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[1;32m 273\u001b[0m \u001b[38;5;66;03m# let the admin agent speak if interrupted\u001b[39;00m\n\u001b[1;32m 274\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m groupchat\u001b[38;5;241m.\u001b[39madmin_name \u001b[38;5;129;01min\u001b[39;00m groupchat\u001b[38;5;241m.\u001b[39magent_names:\n\u001b[1;32m 275\u001b[0m \u001b[38;5;66;03m# admin agent is one of the participants\u001b[39;00m\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:906\u001b[0m, in \u001b[0;36mConversableAgent.generate_reply\u001b[0;34m(self, messages, sender, exclude)\u001b[0m\n\u001b[1;32m 904\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m 905\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_match_trigger(reply_func_tuple[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrigger\u001b[39m\u001b[38;5;124m\"\u001b[39m], sender):\n\u001b[0;32m--> 906\u001b[0m final, reply \u001b[38;5;241m=\u001b[39m \u001b[43mreply_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msender\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreply_func_tuple\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mconfig\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 907\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m final:\n\u001b[1;32m 908\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m reply\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:625\u001b[0m, in \u001b[0;36mConversableAgent.generate_oai_reply\u001b[0;34m(self, messages, sender, config)\u001b[0m\n\u001b[1;32m 622\u001b[0m messages \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oai_messages[sender]\n\u001b[1;32m 624\u001b[0m \u001b[38;5;66;03m# TODO: #1143 handle token limit exceeded error\u001b[39;00m\n\u001b[0;32m--> 625\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 626\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpop\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_oai_system_message\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\n\u001b[1;32m 627\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 628\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m, client\u001b[38;5;241m.\u001b[39mextract_text_or_function_call(response)[\u001b[38;5;241m0\u001b[39m]\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/oai/client.py:247\u001b[0m, in \u001b[0;36mOpenAIWrapper.create\u001b[0;34m(self, **config)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m \u001b[38;5;66;03m# filter is not passed; try the next config\u001b[39;00m\n\u001b[1;32m 246\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 247\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_completions_create\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 248\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m APIError:\n\u001b[1;32m 249\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfig \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m failed\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n", + "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/oai/client.py:327\u001b[0m, in \u001b[0;36mOpenAIWrapper._completions_create\u001b[0;34m(self, client, params)\u001b[0m\n\u001b[1;32m 325\u001b[0m params \u001b[38;5;241m=\u001b[39m params\u001b[38;5;241m.\u001b[39mcopy()\n\u001b[1;32m 326\u001b[0m params[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstream\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 327\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mcompletions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n", + "File \u001b[0;32m~/data-ssd/dynabench/.env/lib/python3.10/site-packages/openai/_utils/_utils.py:299\u001b[0m, in \u001b[0;36mrequired_args..inner..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 297\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required argument: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mquote(missing[\u001b[38;5;241m0\u001b[39m])\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 298\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(msg)\n\u001b[0;32m--> 299\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mTypeError\u001b[0m: Completions.create() got an unexpected keyword argument 'api_base'" + ] + } + ], "source": [ - "new_builder = AgentBuilder(config_path=config_path, agent_model='meta-llama/Llama-2-7b-chat') # use llama-2-7b as agent backbone.\n", + "new_builder = AgentBuilder(config_path=config_path, agent_model='meta-llama/Llama-2-7b-chat-hf', endpoint_building_timeout=1000000000000000000000) # use llama-2-13b-chat as agent backbone.\n", "new_builder.build(building_task, default_llm_config)\n", "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", "new_builder.clear_all_agents()" - ], - "metadata": { - "collapsed": false - }, - "id": "4b16fb43a4dc60f1" + ] }, { "cell_type": "markdown", + "id": "32e0cf8f09eef5cd", + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "source": [ "## GPTs API support\n", "\n", "[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task.\n", "AutoBuild also support GPTs api by adding `use_gpts=True` to the `build()` function." - ], - "metadata": { - "collapsed": false - }, - "id": "32e0cf8f09eef5cd" + ] }, { "cell_type": "code", "execution_count": 11, + "id": "4051c25b2cd1918c", + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-25T19:43:09.367458800Z", + "start_time": "2023-11-25T19:42:15.653608Z" + }, + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [ { "name": "stderr", @@ -771,11 +907,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", + "\u001b[33mSoftware_developer\u001b[0m (to chat_manager):\n", "\n", "To accomplish the task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software development, we will break down the process into the following steps:\n", "\n", @@ -846,8 +982,8 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" ] }, { @@ -861,7 +997,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -894,7 +1030,7 @@ "The paper does not specifically discuss software development applications.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", + "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", "\n", "It appears that the script successfully retrieved information about the latest paper related to GPT-4:\n", "\n", @@ -914,35 +1050,35 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "\n", "\n", @@ -955,44 +1091,39 @@ "new_builder.build(building_task, default_llm_config, use_gpts=True) # Transfer to GPTs API.\n", "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", "new_builder.clear_all_agents()" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-11-25T19:43:09.367458800Z", - "start_time": "2023-11-25T19:42:15.653608Z" - } - }, - "id": "4051c25b2cd1918c" + ] }, { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [], + "id": "57194293da4ae4e1", "metadata": { - "collapsed": false + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } }, - "id": "57194293da4ae4e1" + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.10.11" } }, "nbformat": 4, From c017349e5523843beed6eba5c7c5faa0d549a2ea Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Mon, 27 Nov 2023 12:33:51 +0900 Subject: [PATCH 34/74] switch api_base to base_url --- autogen/agentchat/contrib/agent_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index b87897e369d..b15101cd575 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -191,7 +191,7 @@ def create_agent( else: port = self.agent_procs[server_id][1] - config_list[0]["api_base"] = f"http://{self.host}:{port}/v1" + config_list[0]["base_url"] = f"http://{self.host}:{port}/v1" current_config = llm_config.copy() current_config.update( From ba9f734a4e8fe1efa369b0bad6623b8808220c2a Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Mon, 27 Nov 2023 14:26:42 +0900 Subject: [PATCH 35/74] add details for Step 6; add print in clear_all_agents() --- autogen/agentchat/contrib/agent_builder.py | 9 +- notebook/agentchat_autobuild.ipynb | 340 +++++---------------- 2 files changed, 80 insertions(+), 269 deletions(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index d2805e31a72..c3c264daa6d 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -211,7 +211,7 @@ def create_agent( self.agent_procs_assign[agent_name] = (agent, server_id) return agent - def clear_agent(self, agent_name: str = None, recycle_endpoint: bool = True): + def clear_agent(self, agent_name: str, recycle_endpoint: Optional[bool] = True): """ Clear a specific agent by name. @@ -231,13 +231,15 @@ def clear_agent(self, agent_name: str = None, recycle_endpoint: bool = True): return self.agent_procs[server_id][0].terminate() self.open_ports.append(server_id.split("_")[-1]) + print(f"Agent {agent_name} has been cleared.") - def clear_all_agents(self): + def clear_all_agents(self, recycle_endpoint: Optional[bool] = True): """ Clear all cached agents. """ for agent_name in [agent_name for agent_name in self.agent_procs_assign.keys()]: - self.clear_agent(agent_name) + self.clear_agent(agent_name, recycle_endpoint) + print("All agents have been cleared.") def build( self, @@ -265,6 +267,7 @@ def build( self.building_task = building_task self.default_llm_config = default_llm_config.copy() self.coding = coding + self.agent_configs = [] else: self.building_task = cached_configs["building_task"] self.default_llm_config = cached_configs["default_llm_config"] diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 62d234a9782..5abf6dfbaef 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -4,27 +4,21 @@ "cell_type": "markdown", "id": "c1004af6a7fbfcd8", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "# AutoBuild\n", "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. AgentBuilder has open-source model support, user can specify an open-source model as an agent backbone when initializing." + "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. AutoBuilder also support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html) and [Fastchat](https://github.com/lm-sys/FastChat). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html)." ] }, { "cell_type": "markdown", "id": "ec78dda8e3826d8a", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Requirement\n", @@ -38,10 +32,7 @@ "execution_count": null, "id": "e8e9ae50658be975", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [], "source": [ @@ -52,10 +43,7 @@ "cell_type": "markdown", "id": "7d0e63ab3604bdb9", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Step 1: prepare configuration\n", @@ -64,21 +52,18 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "2505f029423b21ab", "metadata": { - "ExecuteTime": { - "end_time": "2023-11-26T06:42:19.833667600Z", - "start_time": "2023-11-26T06:42:19.818864300Z" - }, "collapsed": false, - "jupyter": { - "outputs_hidden": false + "ExecuteTime": { + "end_time": "2023-11-27T05:14:16.863997100Z", + "start_time": "2023-11-27T05:14:16.846959100Z" } }, "outputs": [], "source": [ - "config_path = '/home/murphy/data-ssd/dynabench/autogen/OAI_CONFIG_LIST_sample' # TODO: modify path\n", + "config_path = 'YOUR CONFIG PATH' # modify path\n", "default_llm_config = {\n", " 'temperature': 0\n", "}" @@ -88,14 +73,11 @@ "cell_type": "markdown", "id": "c2d6586c68fa425b", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Step 2: create a AgentBuilder\n", - "Create a `AgentBuilder` with the specified `config_path`. AgentBuilder will use GPT-4 in default to complete the whole process, you can also change the `builder_model` to other OpenAI model if you want." + "Create a `AgentBuilder` with the specified `config_path`. AgentBuilder will use GPT-4 in default to complete the whole process, you can also change the `builder_model` to other OpenAI model if you want. You can also specify a OpenAI or open-source LLM as agent backbone, see blog for more details." ] }, { @@ -107,26 +89,20 @@ "end_time": "2023-11-26T06:42:24.270587200Z", "start_time": "2023-11-26T06:42:22.943534500Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [], "source": [ "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", "\n", - "builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview')" + "builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview', agent_model='gpt-4-1106-preview')" ] }, { "cell_type": "markdown", "id": "2e6a655fb6618324", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Step 3: specify a building task\n", @@ -142,10 +118,7 @@ "ExecuteTime": { "start_time": "2023-11-26T06:42:20.859520700Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [], "source": [ @@ -156,10 +129,7 @@ "cell_type": "markdown", "id": "5782dd5ecb6c217a", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Step 4: build group chat agents\n", @@ -179,10 +149,7 @@ "end_time": "2023-11-26T06:42:25.995470100Z", "start_time": "2023-11-26T06:42:25.917253700Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [ { @@ -209,10 +176,7 @@ "cell_type": "markdown", "id": "e00dd99880a4bf7b", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Step 5: execute task\n", @@ -228,22 +192,19 @@ "end_time": "2023-11-25T19:27:43.776416600Z", "start_time": "2023-11-25T19:27:40.439581500Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mSoftware_developer\u001b[0m (to chat_manager):\n", + "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", "\n", "To accomplish this task, we will follow these steps:\n", "\n", @@ -286,8 +247,8 @@ "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" ] }, { @@ -301,7 +262,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -332,7 +293,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", "\n", @@ -364,42 +325,42 @@ "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", @@ -415,14 +376,11 @@ "cell_type": "markdown", "id": "22a30e4b4297edd1", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Step 6 (Optional): clear all agents and prepare for the next task\n", - "You can clear all agents generated in this task by the following code if your task is completed or the next task is largely different from the current task." + "You can clear all agents generated in this task by the following code if your task is completed or the next task is largely different from the current task. If the agent's backbone is an open-source LLM, this process will also shutdown the endpoint server. If necessary, you can use `recycle_endpoint=False` to retain the previous open-source LLMs' endpoint server." ] }, { @@ -434,24 +392,18 @@ "end_time": "2023-11-25T19:29:39.150365Z", "start_time": "2023-11-25T19:29:39.147791300Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [], "source": [ - "builder.clear_all_agents()" + "builder.clear_all_agents(recycle_endpoint=True)" ] }, { "cell_type": "markdown", "id": "bbb098638a086898", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "## Save & load configs\n", @@ -496,10 +448,7 @@ "end_time": "2023-11-25T19:41:44.198099300Z", "start_time": "2023-11-25T19:41:44.194085500Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [ { @@ -518,10 +467,7 @@ "cell_type": "markdown", "id": "a35620c10ee42be", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ "After that, you can load the saved config and skip the building process. AutoBuilder will create agents with those information without prompting the builder manager." @@ -536,22 +482,19 @@ "end_time": "2023-11-25T19:41:48.044519600Z", "start_time": "2023-11-25T19:41:45.661666200Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mSoftware_developer\u001b[0m (to chat_manager):\n", + "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", "\n", "To accomplish this task, we will follow these steps:\n", "\n", @@ -594,8 +537,8 @@ "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" ] }, { @@ -609,7 +552,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -640,7 +583,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", "\n", @@ -672,41 +615,41 @@ "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", "It seems there might be some confusion or an issue with the user console. If you have any further questions or need assistance with another task, please let me know, and I'll be happy to help. If you're done, no further action is needed. Thank you!\n", "\n", @@ -720,143 +663,14 @@ "new_builder.clear_all_agents()" ] }, - { - "cell_type": "markdown", - "id": "f6200da25a755df", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "## Open-source LLMs support\n", - "AutoBuilder support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html).\n", - "If you want to use open-source model, `vllm>=0.2.0` and `fastchat>=0.2.30` is required. You can install vLLM and fastchat by the following command:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f45061ba99c1d5", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "!pip install vllm\n", - "!pip install \"fschat[model_worker,webui]\"" - ] - }, - { - "cell_type": "markdown", - "id": "f7af6180af22bec9", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "source": [ - "After satisfying the requirements, you can add an open-source LLM's huggingface repository to the config file,\n", - "```json\n", - "// Add the LLM's huggingface repo to your config file\n", - "[\n", - " ...\n", - " {\n", - " \"model\": \"meta-llama/Llama-2-13b-chat-hf\",\n", - " \"api_key\": \"EMPTY\"\n", - " }\n", - "]\n", - "```\n", - "and specify it when initializing AgentBuilder. \n", - "AgentBuilder will automatically set up an endpoint server for open-source LLM, make sure you have sufficient GPUs resources." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "4b16fb43a4dc60f1", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating agents...\n", - "Data_scientist,Research_analyst,Software_developer are generated.\n", - "Preparing configuration for Data_scientist...\n", - "Preparing configuration for Research_analyst...\n", - "Preparing configuration for Software_developer...\n", - "Creating agent Data_scientist with backbone meta-llama/Llama-2-7b-chat-hf...\n", - "b\"INFO 11-26 14:01:39 llm_engine.py:72] Initializing an LLM engine with config: model='meta-llama/Llama-2-7b-chat-hf', tokenizer='meta-llama/Llama-2-7b-chat-hf', tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, quantization=None, seed=0)\\n\"\n", - "b\"INFO 11-26 14:01:39 tokenizer.py:31] For some LLaMA V1 models, initializing the fast tokenizer may take a long time. To reduce the initialization time, consider using 'hf-internal-testing/llama-tokenizer' instead of the original tokenizer.\\n\"\n", - "b'INFO 11-26 14:01:45 llm_engine.py:207] # GPU blocks: 3803, # CPU blocks: 512\\n'\n", - "b\"INFO 11-26 14:01:46 tokenizer.py:31] For some LLaMA V1 models, initializing the fast tokenizer may take a long time. To reduce the initialization time, consider using 'hf-internal-testing/llama-tokenizer' instead of the original tokenizer.\\n\"\n", - "b'INFO: Started server process [340198]\\n'\n", - "b'INFO: Waiting for application startup.\\n'\n", - "b'INFO: Application startup complete.\\n'\n", - "b'INFO: Uvicorn running on http://localhost:65534 (Press CTRL+C to quit)\\n'\n", - "Running meta-llama/Llama-2-7b-chat-hf on http://localhost:65534 with tensor parallel size 1.\n", - "Creating agent Research_analyst with backbone meta-llama/Llama-2-7b-chat-hf...\n", - "Creating agent Software_developer with backbone meta-llama/Llama-2-7b-chat-hf...\n", - "Adding user console proxy...\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", - "\n", - "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", - "\n", - "--------------------------------------------------------------------------------\n" - ] - }, - { - "ename": "TypeError", - "evalue": "Completions.create() got an unexpected keyword argument 'api_base'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[5], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m new_builder \u001b[38;5;241m=\u001b[39m AgentBuilder(config_path\u001b[38;5;241m=\u001b[39mconfig_path, agent_model\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmeta-llama/Llama-2-7b-chat-hf\u001b[39m\u001b[38;5;124m'\u001b[39m, endpoint_building_timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1000000000000000000000\u001b[39m) \u001b[38;5;66;03m# use llama-2-13b-chat as agent backbone.\u001b[39;00m\n\u001b[1;32m 2\u001b[0m new_builder\u001b[38;5;241m.\u001b[39mbuild(building_task, default_llm_config)\n\u001b[0;32m----> 3\u001b[0m \u001b[43mnew_builder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mFind a latest paper about gpt-4 on arxiv and find its potential applications in software.\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m new_builder\u001b[38;5;241m.\u001b[39mclear_all_agents()\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/contrib/agent_builder.py:421\u001b[0m, in \u001b[0;36mAgentBuilder.start\u001b[0;34m(self, task, max_round, init_messages)\u001b[0m\n\u001b[1;32m 416\u001b[0m manager \u001b[38;5;241m=\u001b[39m autogen\u001b[38;5;241m.\u001b[39mGroupChatManager(\n\u001b[1;32m 417\u001b[0m groupchat\u001b[38;5;241m=\u001b[39mgroup_chat, llm_config\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroup_chat_manager_config, system_message\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmanager_system_message\n\u001b[1;32m 418\u001b[0m )\n\u001b[1;32m 420\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minitiate_agent_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUser_console_and_Python_code_interpreter\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muser_proxy \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 421\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muser_proxy\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minitiate_chat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmanager\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessage\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtask\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 422\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 423\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent \u001b[38;5;129;01min\u001b[39;00m agent_list:\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:550\u001b[0m, in \u001b[0;36mConversableAgent.initiate_chat\u001b[0;34m(self, recipient, clear_history, silent, **context)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Initiate a chat with the recipient agent.\u001b[39;00m\n\u001b[1;32m 537\u001b[0m \n\u001b[1;32m 538\u001b[0m \u001b[38;5;124;03mReset the consecutive auto reply counter.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 547\u001b[0m \u001b[38;5;124;03m \"message\" needs to be provided if the `generate_init_message` method is not overridden.\u001b[39;00m\n\u001b[1;32m 548\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 549\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_chat(recipient, clear_history)\n\u001b[0;32m--> 550\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_init_message\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrecipient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msilent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msilent\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:348\u001b[0m, in \u001b[0;36mConversableAgent.send\u001b[0;34m(self, message, recipient, request_reply, silent)\u001b[0m\n\u001b[1;32m 346\u001b[0m valid \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_append_oai_message(message, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124massistant\u001b[39m\u001b[38;5;124m\"\u001b[39m, recipient)\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m valid:\n\u001b[0;32m--> 348\u001b[0m \u001b[43mrecipient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreceive\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest_reply\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msilent\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 349\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 350\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 351\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMessage can\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt be converted into a valid ChatCompletion message. Either content or function_call must be provided.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 352\u001b[0m )\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:481\u001b[0m, in \u001b[0;36mConversableAgent.receive\u001b[0;34m(self, message, sender, request_reply, silent)\u001b[0m\n\u001b[1;32m 479\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m request_reply \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m request_reply \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreply_at_receive[sender] \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m:\n\u001b[1;32m 480\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[0;32m--> 481\u001b[0m reply \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_reply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchat_messages\u001b[49m\u001b[43m[\u001b[49m\u001b[43msender\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msender\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 482\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m reply \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 483\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msend(reply, sender, silent\u001b[38;5;241m=\u001b[39msilent)\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:906\u001b[0m, in \u001b[0;36mConversableAgent.generate_reply\u001b[0;34m(self, messages, sender, exclude)\u001b[0m\n\u001b[1;32m 904\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m 905\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_match_trigger(reply_func_tuple[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrigger\u001b[39m\u001b[38;5;124m\"\u001b[39m], sender):\n\u001b[0;32m--> 906\u001b[0m final, reply \u001b[38;5;241m=\u001b[39m \u001b[43mreply_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msender\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreply_func_tuple\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mconfig\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 907\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m final:\n\u001b[1;32m 908\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m reply\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/groupchat.py:271\u001b[0m, in \u001b[0;36mGroupChatManager.run_chat\u001b[0;34m(self, messages, sender, config)\u001b[0m\n\u001b[1;32m 269\u001b[0m speaker \u001b[38;5;241m=\u001b[39m groupchat\u001b[38;5;241m.\u001b[39mselect_speaker(speaker, \u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m 270\u001b[0m \u001b[38;5;66;03m# let the speaker speak\u001b[39;00m\n\u001b[0;32m--> 271\u001b[0m reply \u001b[38;5;241m=\u001b[39m \u001b[43mspeaker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_reply\u001b[49m\u001b[43m(\u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 272\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[1;32m 273\u001b[0m \u001b[38;5;66;03m# let the admin agent speak if interrupted\u001b[39;00m\n\u001b[1;32m 274\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m groupchat\u001b[38;5;241m.\u001b[39madmin_name \u001b[38;5;129;01min\u001b[39;00m groupchat\u001b[38;5;241m.\u001b[39magent_names:\n\u001b[1;32m 275\u001b[0m \u001b[38;5;66;03m# admin agent is one of the participants\u001b[39;00m\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:906\u001b[0m, in \u001b[0;36mConversableAgent.generate_reply\u001b[0;34m(self, messages, sender, exclude)\u001b[0m\n\u001b[1;32m 904\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[1;32m 905\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_match_trigger(reply_func_tuple[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrigger\u001b[39m\u001b[38;5;124m\"\u001b[39m], sender):\n\u001b[0;32m--> 906\u001b[0m final, reply \u001b[38;5;241m=\u001b[39m \u001b[43mreply_func\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msender\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msender\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreply_func_tuple\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mconfig\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 907\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m final:\n\u001b[1;32m 908\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m reply\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/agentchat/conversable_agent.py:625\u001b[0m, in \u001b[0;36mConversableAgent.generate_oai_reply\u001b[0;34m(self, messages, sender, config)\u001b[0m\n\u001b[1;32m 622\u001b[0m messages \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_oai_messages[sender]\n\u001b[1;32m 624\u001b[0m \u001b[38;5;66;03m# TODO: #1143 handle token limit exceeded error\u001b[39;00m\n\u001b[0;32m--> 625\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 626\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpop\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcontext\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_oai_system_message\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\n\u001b[1;32m 627\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 628\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m, client\u001b[38;5;241m.\u001b[39mextract_text_or_function_call(response)[\u001b[38;5;241m0\u001b[39m]\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/oai/client.py:247\u001b[0m, in \u001b[0;36mOpenAIWrapper.create\u001b[0;34m(self, **config)\u001b[0m\n\u001b[1;32m 245\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m \u001b[38;5;66;03m# filter is not passed; try the next config\u001b[39;00m\n\u001b[1;32m 246\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 247\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_completions_create\u001b[49m\u001b[43m(\u001b[49m\u001b[43mclient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 248\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m APIError:\n\u001b[1;32m 249\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfig \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m failed\u001b[39m\u001b[38;5;124m\"\u001b[39m, exc_info\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n", - "File \u001b[0;32m/mnt/sdc1/dynabench/autogen/autogen/oai/client.py:327\u001b[0m, in \u001b[0;36mOpenAIWrapper._completions_create\u001b[0;34m(self, client, params)\u001b[0m\n\u001b[1;32m 325\u001b[0m params \u001b[38;5;241m=\u001b[39m params\u001b[38;5;241m.\u001b[39mcopy()\n\u001b[1;32m 326\u001b[0m params[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstream\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[0;32m--> 327\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mcompletions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n", - "File \u001b[0;32m~/data-ssd/dynabench/.env/lib/python3.10/site-packages/openai/_utils/_utils.py:299\u001b[0m, in \u001b[0;36mrequired_args..inner..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 297\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMissing required argument: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mquote(missing[\u001b[38;5;241m0\u001b[39m])\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 298\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(msg)\n\u001b[0;32m--> 299\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[0;31mTypeError\u001b[0m: Completions.create() got an unexpected keyword argument 'api_base'" - ] - } - ], - "source": [ - "new_builder = AgentBuilder(config_path=config_path, agent_model='meta-llama/Llama-2-7b-chat-hf', endpoint_building_timeout=1000000000000000000000) # use llama-2-13b-chat as agent backbone.\n", - "new_builder.build(building_task, default_llm_config)\n", - "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", - "new_builder.clear_all_agents()" - ] - }, { "cell_type": "markdown", "id": "32e0cf8f09eef5cd", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "source": [ - "## GPTs API support\n", + "## Use GPTs\n", "\n", "[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task.\n", "AutoBuild also support GPTs api by adding `use_gpts=True` to the `build()` function." @@ -871,10 +685,7 @@ "end_time": "2023-11-25T19:43:09.367458800Z", "start_time": "2023-11-25T19:42:15.653608Z" }, - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [ { @@ -907,11 +718,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mSoftware_developer\u001b[0m (to chat_manager):\n", + "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", "\n", "To accomplish the task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software development, we will break down the process into the following steps:\n", "\n", @@ -982,8 +793,8 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" ] }, { @@ -997,7 +808,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -1030,7 +841,7 @@ "The paper does not specifically discuss software development applications.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mResearch_analyst\u001b[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", "It appears that the script successfully retrieved information about the latest paper related to GPT-4:\n", "\n", @@ -1050,35 +861,35 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "\n", "\n", @@ -1098,10 +909,7 @@ "execution_count": null, "id": "57194293da4ae4e1", "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } + "collapsed": false }, "outputs": [], "source": [] From 38ad6dbc6bbfd9a068a871e1551a661b266e1df0 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Mon, 27 Nov 2023 14:46:01 +0900 Subject: [PATCH 36/74] Change description of Step 5 --- notebook/agentchat_autobuild.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 5abf6dfbaef..d612df071e4 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -180,7 +180,7 @@ }, "source": [ "## Step 5: execute task\n", - "Use agents generated in `build()` to complete the task collaboratively in a group chat." + "Let agents generated in `build()` to complete the task collaboratively in a group chat." ] }, { From ece286b63dfa7b26ef35d8d841a0a802b3a46d74 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Mon, 27 Nov 2023 14:48:55 +0900 Subject: [PATCH 37/74] delete ASSISTANT_CONFIG_LIST --- ASSISTANT_CONFIG_LIST | 251 ------------------------------------------ 1 file changed, 251 deletions(-) delete mode 100644 ASSISTANT_CONFIG_LIST diff --git a/ASSISTANT_CONFIG_LIST b/ASSISTANT_CONFIG_LIST deleted file mode 100644 index 760c009ac28..00000000000 --- a/ASSISTANT_CONFIG_LIST +++ /dev/null @@ -1,251 +0,0 @@ -{ - "gpt-4": { - "base": "GPT", - "type": ["chat", "instruct", "code"], - "language": ["en", "cn", "de", "es", "fr", "jp", "kr"], - "params": "online", - "context": "8192", - "release": "2023", - "benchmark": {}, - "description": "(High cost) A set of models that improve on GPT-3.5 and can understand as well as generate natural language or code." - }, - "gpt-3.5-turbo": { - "base": "GPT", - "type": ["chat", "instruct", "code"], - "language": ["en", "cn", "de", "es", "fr", "jp", "kr"], - "params": "online", - "context": "4096", - "release": "2023", - "benchmark": {}, - "description": "(High cost) A set of models that improve on GPT-3 and can understand as well as generate natural language or code." - }, - "meta-llama/Llama-2-7b-chat-hf": { - "base": "LLAMA", - "type": ["chat", "instruct", "code"], - "language": ["en"], - "params": "7b", - "context": "4096", - "precision": "fp16", - "release": "2023", - "benchmark": { - "ARC": 52.9, - "HellaSwag": 78.55, - "MMLU": 48.32, - "TruthfulQA": 45.57, - "avg": 56.34 - }, - "description": "Llama 2 is a collection of pretrained and fine-tuned generative text models ranging in scale from 7 billion to 70 billion parameters. This is the description for the 7B fine-tuned model, optimized for dialogue use cases and converted for the Hugging Face Transformers format." - }, - "meta-llama/Llama-2-13b-chat-hf": { - "base": "LLAMA", - "type": ["chat", "instruct", "code"], - "language": ["en"], - "params": "13b", - "context": "4096", - "precision": "fp16", - "release": "2023", - "benchmark": { - "ARC": 59.04, - "HellaSwag": 81.94, - "MMLU": 54.64, - "TruthfulQA": 44.12, - "avg": 59.93 - }, - "description": "Llama 2 is a collection of pretrained and fine-tuned generative text models ranging in scale from 7 billion to 70 billion parameters. This is the description for the 13B fine-tuned model, optimized for dialogue use cases and converted for the Hugging Face Transformers format." - }, - "meta-llama/Llama-2-70b-chat-hf": { - "model_name": "LLAMA", - "type": ["chat", "instruct", "code"], - "language": ["en"], - "params": "70b", - "context": "4096", - "precision": "fp16", - "release": "2023", - "benchmark": { - "ARC": 64.59, - "HellaSwag": 85.88, - "MMLU": 63.91, - "TruthfulQA": 52.8, - "avg": 66.8 - }, - "description": "Llama 2 is a collection of pretrained and fine-tuned generative text models ranging in scale from 7 billion to 70 billion parameters. This is the description for the 70B fine-tuned model, optimized for dialogue use cases and converted for the Hugging Face Transformers format." - }, - "tiiuae/falcon-7b-instruct": { - "base": "Falcon", - "type": ["chat", "instruct", "code"], - "language": ["en", "fr"], - "params": "7b", - "context": "2048", - "precision": "fp16", - "release": "2023", - "benchmark": {}, - "description": "Falcon-7B-Instruct is a 7B parameters causal decoder-only model built by TII based on Falcon-7B and finetuned on a mixture of chat/instruct datasets." - }, - "tiiuae/falcon-40b-instruct": { - "base": "Falcon", - "type": ["chat", "instruct", "code"], - "language": ["en", "fr"], - "params": "40b", - "context": "2048", - "precision": "fp16", - "release": "2023", - "benchmark": { - "ARC": 61.6, - "HellaSwag": 84.31, - "MMLU": 55.45, - "TruthfulQA": 52.52, - "avg": 63.47 - }, - "description": "Falcon-40B-Instruct is a 40B parameters causal decoder-only model built by TII based on Falcon-40B and finetuned on a mixture of Baize." - }, - "tiiuae/falcon-180B-chat": { - "base": "Falcon", - "type": ["chat", "instruct", "code"], - "language": ["en", "de", "es", "fr"], - "params": "180b", - "context": "2048", - "precision": "fp16", - "release": "2023", - "benchmark": {}, - "description": "Falcon-180B-Chat is a 180B parameters causal decoder-only model built by TII based on Falcon-180B and finetuned on a mixture of Ultrachat, Platypus and Airoboros." - }, - "FlagAlpha/Llama2-Chinese-7b-Chat": { - "base": "LLAMA", - "type": ["chat", "instruct", "code"], - "language": ["cn", "en"], - "params": "7b", - "context": "4096", - "precision": "fp16", - "release": "2023", - "benchmark": { - "ARC": 52.39, - "HellaSwag": 77.52, - "MMLU": 47.72, - "TruthfulQA": 46.87, - "avg": 56.13 - }, - "description": "Since Llama2 itself has weak Chinese alignment, we use the Chinese instruction set and fine-tune for meta-llama/Llama-2-7b-chat-hf to make it capable of strong Chinese dialog." - }, - "FlagAlpha/Llama2-Chinese-13b-Chat": { - "base": "LLAMA", - "type": ["chat", "instruct", "code"], - "language": ["cn", "en"], - "params": "13b", - "context": "4096", - "precision": "fp16", - "release": "2023", - "benchmark": { - "ARC": 55.97, - "HellaSwag": 82.05, - "MMLU": 54.74, - "TruthfulQA": 48.9, - "avg": 60.41 - }, - "description": "Since Llama2 itself has weak Chinese alignment, we use the Chinese instruction set and fine-tune for meta-llama/Llama-2-13b-chat-hf to make it capable of strong Chinese dialog." - }, - "baichuan-inc/Baichuan2-7B-Chat": { - "base": "Baichuan", - "type": ["chat", "instruct", "code"], - "language": ["cn", "en"], - "params": "7b", - "context": "4096", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Baichuan 2 (7b) is the new generation of large-scale open-source language models launched by Baichuan Intelligence inc.. It is trained on a high-quality corpus with 2.6 trillion tokens and has achieved the promising performance in authoritative Chinese and English benchmarks of the same size." - }, - "baichuan-inc/Baichuan2-13B-Chat": { - "base": "Baichuan", - "type": ["chat", "instruct", "code"], - "language": ["cn", "en"], - "params": "13b", - "context": "4096", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Baichuan 2 (13b) is the new generation of large-scale open-source language models launched by Baichuan Intelligence inc.. It is trained on a high-quality corpus with 2.6 trillion tokens and has achieved the promising performance in authoritative Chinese and English benchmarks of the same size." - }, - "elyza/ELYZA-japanese-Llama-2-7b-instruct": { - "base": "LLAMA", - "type": ["instruct", "code"], - "language": ["jp", "en"], - "params": "7b", - "context": "4096", - "precision": "fp16", - "release": "2023", - "benchmark": { - "ARC": 53.16, - "HellaSwag": 78.25, - "MMLU": 47.07, - "TruthfulQA": 39.08, - "avg": 54.39 - }, - "description": "ELYZA-japanese-Llama-2-7b is a model based on Llama2 with additional find-tuning to extend Japanese language capabilities." - }, - "codellama/CodeLlama-7b-Instruct-hf": { - "base": "LLAMA", - "type": ["code", "instruct"], - "language": ["en"], - "params": "34b", - "context": "16384", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Code Llama (7b) is designed for general code synthesis and understanding." - }, - "codellama/CodeLlama-13b-Instruct-hf": { - "base": "LLAMA", - "type": ["code", "instruct"], - "language": ["en"], - "params": "34b", - "context": "16384", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Code Llama (13b) is designed for general code synthesis and understanding." - }, - "codellama/CodeLlama-34b-Instruct-hf": { - "base": "LLAMA", - "type": ["code", "instruct"], - "language": ["en"], - "params": "34b", - "context": "16384", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Code Llama (34b) is designed for general code synthesis and understanding." - }, - "codellama/CodeLlama-7b-Python-hf": { - "base": "LLAMA", - "type": ["code", "instruct"], - "language": ["en"], - "params": "7b", - "context": "16384", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Code Llama (7b, python) is designed specifically for generating python code." - }, - "codellama/CodeLlama-13b-Python-hf": { - "base": "LLAMA", - "type": ["code", "instruct"], - "language": ["en"], - "params": "13b", - "context": "16384", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Code Llama (13b, python) is designed specifically for generating python code." - }, - "codellama/CodeLlama-34b-Python-hf": { - "base": "LLAMA", - "type": ["code", "instruct"], - "language": ["en"], - "params": "34b", - "context": "16384", - "precision": "bfp16", - "release": "2023", - "benchmark": {}, - "description": "Code Llama (34b, python) is designed specifically for generating python code." - } -} \ No newline at end of file From 0dadaaf1e3de1f0cac51bdea37f4855bbf5d6b1f Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Sun, 26 Nov 2023 22:00:15 -0800 Subject: [PATCH 38/74] add Linxin to blog authors --- website/blog/authors.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/website/blog/authors.yml b/website/blog/authors.yml index a487086cb22..7f6f7dc482b 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -87,3 +87,9 @@ narabzad: title: PhD student at the University of Waterloo url: https://www.negara.me/ image_url: https://github.com/Narabzad.png + +LinxinS97: + name: Linxin Song + title: MS student at Waseda University + url: https://linxins97.github.io/ + image_url: https://github.com/LinxinS97.png From ae9737bf65924f8a4280387f2577c8b9d4d35310 Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Sun, 26 Nov 2023 22:05:48 -0800 Subject: [PATCH 39/74] add blog --- website/blog/2023-11-26-Agent-AutoBuild/index.mdx | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 website/blog/2023-11-26-Agent-AutoBuild/index.mdx diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx new file mode 100644 index 00000000000..2fccb484753 --- /dev/null +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -0,0 +1,13 @@ +--- +title: Agent AutoBuild: Automatically Building Multi-agent Systems +authors: + - LinxinS97 + - jieyuz2 +tags: [LLM] +--- + +![system](img/system.png) + +**TL;DR:** + + From 02f0d81c2dcf1a9cf9e72783475021199746d65a Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Sun, 26 Nov 2023 22:11:00 -0800 Subject: [PATCH 40/74] Update index.mdx --- website/blog/2023-11-26-Agent-AutoBuild/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index 2fccb484753..94d9a6a944f 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -1,5 +1,5 @@ --- -title: Agent AutoBuild: Automatically Building Multi-agent Systems +title: Agent AutoBuild - Automatically Building Multi-agent Systems authors: - LinxinS97 - jieyuz2 From 0cf7c3ae345964727a2339eac6ed7bfd3f53c062 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 03:40:33 +0900 Subject: [PATCH 41/74] add test; add user proxy constrain; change show case in notebook --- autogen/agentchat/contrib/agent_builder.py | 37 +- notebook/agentchat_autobuild.ipynb | 806 +++++++++--------- test/agentchat/contrib/test_autobuder.py | 97 +++ ...nfig_eb1be857faa608aeb4c5af11fe4ab245.json | 25 + 4 files changed, 563 insertions(+), 402 deletions(-) create mode 100644 test/agentchat/contrib/test_autobuder.py create mode 100644 test/test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index c3c264daa6d..dd92deb79fc 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -56,7 +56,7 @@ class AgentBuilder: # The modified requirement should not contain the code interpreter skill. # Coding skill is limited to Python. # Your answer should omit the word "REQUIREMENT". - # Your answer should include the description of the behavior when the task complete (user's need has been satisfied). + # Your should let them reply "TERMINATE" in the end when the task complete (user's need has been satisfied). """ def __init__( @@ -65,7 +65,7 @@ def __init__( builder_model: Optional[str] = "gpt-4-1106-preview", agent_model: Optional[str] = "gpt-4-1106-preview", host: Optional[str] = "localhost", - endpoint_building_timeout: Optional[int] = 180, + endpoint_building_timeout: Optional[int] = 600, ): """ Args: @@ -322,17 +322,19 @@ def build( self.agent_configs.append( {"name": agent_name_list[i], "model": self.agent_model, "system_message": agent_sys_msg_list[i]} ) - print(f"Creating agent {agent_name_list[i]} with backbone {self.agent_model}...") - self.create_agent( - agent_name_list[i], - self.agent_model, - self.default_llm_config, - system_message=agent_sys_msg_list[i], - use_gpts=use_gpts, - **kwargs, - ) self.manager_system_message = "Group chat manager." + for config in self.agent_configs: + print(f"Creating agent {config['name']} with backbone {config['model']}...") + self.create_agent( + config["name"], + config["model"], + self.default_llm_config, + system_message=config["system_message"], + use_gpts=use_gpts, + **kwargs, + ) + if self.coding is None: resp = ( build_manager.create( @@ -347,6 +349,7 @@ def build( print("Adding user console proxy...") self.user_proxy = autogen.UserProxyAgent( name="User_console_and_Python_code_interpreter", + is_termination_msg=lambda x: "TERMINATE" in x.get("content"), system_message="User console with a python code interpreter interface.", code_execution_config={"last_n_messages": 2, "work_dir": "groupchat"}, human_input_mode="NEVER", @@ -385,22 +388,26 @@ def save(self, filepath: Optional[str] = None) -> str: return filepath - def load(self, filepath: str): + def load(self, filepath: str, use_gpts: Optional[bool] = False): """ Load building configs and call the build function to complete building without calling online LLMs' api. Args: filepath: filepath for the save config. + use_gpts: if true, build function will use GPTs api to build agents. """ if os.path.isfile(filepath): + print(f"Loding config from {filepath}") cached_configs = json.load(open(filepath)) - self.build(cached_configs=cached_configs) + self.build(cached_configs=cached_configs, use_gpts=use_gpts) else: raise FileNotFoundError(f"Config file {filepath} does not exist.") return self - def start(self, task: str, max_round: Optional[int] = 12, init_messages: Optional[List[dict]] = []): + def start( + self, task: str, max_round: Optional[int] = 12, init_messages: Optional[List[dict]] = [] + ) -> Tuple[autogen.GroupChat, autogen.GroupChatManager]: """ Start a group chat task solving process with built config. @@ -426,3 +433,5 @@ def start(self, task: str, max_round: Optional[int] = 12, init_messages: Optiona for agent in agent_list: if self.initiate_agent_name == agent.name: agent.initiate_chat(manager, message=task) + + return group_chat, manager diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index d612df071e4..0793bd15a44 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -57,13 +57,13 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-27T05:14:16.863997100Z", - "start_time": "2023-11-27T05:14:16.846959100Z" + "end_time": "2023-11-28T09:23:18.987795800Z", + "start_time": "2023-11-28T09:23:18.977219Z" } }, "outputs": [], "source": [ - "config_path = 'YOUR CONFIG PATH' # modify path\n", + "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # modify path\n", "default_llm_config = {\n", " 'temperature': 0\n", "}" @@ -82,14 +82,14 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "bfa67c771a0fed37", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-26T06:42:24.270587200Z", - "start_time": "2023-11-26T06:42:22.943534500Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:23:19.902186400Z", + "start_time": "2023-11-28T09:23:18.979228600Z" + } }, "outputs": [], "source": [ @@ -112,13 +112,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "68315f6ec912c58a", "metadata": { + "collapsed": false, "ExecuteTime": { - "start_time": "2023-11-26T06:42:20.859520700Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:23:21.419250500Z", + "start_time": "2023-11-28T09:23:21.413220200Z" + } }, "outputs": [], "source": [ @@ -145,11 +146,11 @@ "execution_count": 4, "id": "ab490fdbe46c0473", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-26T06:42:25.995470100Z", - "start_time": "2023-11-26T06:42:25.917253700Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:23:23.564327400Z", + "start_time": "2023-11-28T09:23:23.488457300Z" + } }, "outputs": [ { @@ -157,13 +158,13 @@ "output_type": "stream", "text": [ "Generating agents...\n", - "Data_scientist,Research_analyst,Software_developer are generated.\n", + "Data_scientist,Domain_expert,Software_engineer are generated.\n", "Preparing configuration for Data_scientist...\n", - "Preparing configuration for Research_analyst...\n", - "Preparing configuration for Software_developer...\n", + "Preparing configuration for Domain_expert...\n", + "Preparing configuration for Software_engineer...\n", "Creating agent Data_scientist with backbone gpt-4-1106-preview...\n", - "Creating agent Research_analyst with backbone gpt-4-1106-preview...\n", - "Creating agent Software_developer with backbone gpt-4-1106-preview...\n", + "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", + "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", "Adding user console proxy...\n" ] } @@ -185,14 +186,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "7d52e3d9a1bf91cb", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:27:43.776416600Z", - "start_time": "2023-11-25T19:27:40.439581500Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:23:27.331284800Z", + "start_time": "2023-11-28T09:23:25.264410200Z" + } }, "outputs": [ { @@ -204,51 +205,112 @@ "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", - "\n", - "To accomplish this task, we will follow these steps:\n", + "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", "\n", - "1. **Search arXiv for the latest papers on GPT-4**: We will use the `arxiv` Python library to search for the most recent papers related to GPT-4. If the `arxiv` library is not installed, you will need to install it using `pip install arxiv`.\n", + "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers that match the query. We'll write a Python script that uses the `requests` library to make a GET request to the arXiv API with the appropriate query parameters. The script will parse the response to find the most recent paper related to GPT-4 and then print out the title, authors, summary, and potential applications in software.\n", "\n", - "2. **Extract Information**: From the search results, we will extract the title, authors, summary, and publication date of the latest paper.\n", + "First, ensure you have the `requests` library installed. If not, you can install it using pip:\n", "\n", - "3. **Analyze for Potential Applications in Software Development**: We will read through the summary to identify mentions of potential applications in software development.\n", - "\n", - "4. **Present Findings**: We will print out the relevant information.\n", + "```bash\n", + "pip install requests\n", + "```\n", "\n", - "Let's start with step 1. Here is the Python code to search for the latest papers on GPT-4 on arXiv:\n", + "Now, here's the Python script to find the latest paper about GPT-4 on arXiv:\n", "\n", "```python\n", - "import arxiv\n", + "import requests\n", + "from datetime import datetime\n", + "import xml.etree.ElementTree as ET\n", "\n", - "# Define the search query and parameters\n", - "search_query = 'all:\"GPT-4\"'\n", - "sort_by = arxiv.SortCriterion.SubmittedDate\n", - "sort_order = arxiv.SortOrder.Descending\n", + "# Function to search arXiv for papers related to GPT-4\n", + "def search_arxiv(query):\n", + " # Define the URL and parameters for the arXiv API request\n", + " url = 'http://export.arxiv.org/api/query'\n", + " params = {\n", + " 'search_query': query,\n", + " 'sortBy': 'submittedDate',\n", + " 'sortOrder': 'descending',\n", + " 'max_results': 1\n", + " }\n", + " \n", + " # Make the GET request to the arXiv API\n", + " response = requests.get(url, params=params)\n", + " \n", + " # Check if the request was successful\n", + " if response.status_code == 200:\n", + " return response.text\n", + " else:\n", + " print(f\"Error: {response.status_code}\")\n", + " return None\n", "\n", - "# Perform the search\n", - "search = arxiv.Search(\n", - " query=search_query,\n", - " max_results=1,\n", - " sort_by=sort_by,\n", - " sort_order=sort_order\n", - ")\n", + "# Function to parse the XML response and extract paper details\n", + "def parse_arxiv_response(xml_response):\n", + " # Parse the XML response\n", + " root = ET.fromstring(xml_response)\n", + " \n", + " # Find the entry element, which contains the paper information\n", + " entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", + " \n", + " # Extract the title, authors, summary, and published date\n", + " title = entry.find('{http://www.w3.org/2005/Atom}title').text.strip()\n", + " authors = [author.find('{http://www.w3.org/2005/Atom}name').text for author in entry.findall('{http://www.w3.org/2005/Atom}author')]\n", + " summary = entry.find('{http://www.w3.org/2005/Atom}summary').text.strip()\n", + " published_date = entry.find('{http://www.w3.org/2005/Atom}published').text\n", + " \n", + " return {\n", + " 'title': title,\n", + " 'authors': authors,\n", + " 'summary': summary,\n", + " 'published_date': published_date\n", + " }\n", "\n", - "# Fetch the result\n", - "for result in search.results():\n", - " print(\"Title:\", result.title)\n", - " print(\"Authors:\", \", \".join(author.name for author in result.authors))\n", - " print(\"Abstract:\", result.summary)\n", - " print(\"Publication Date:\", result.published)\n", - " print(\"URL:\", result.entry_id)\n", - " break # Since we only want the latest paper, we break after the first result\n", + "# Main function to execute the script\n", + "def main():\n", + " # Search for the latest paper on GPT-4\n", + " query = 'all:gpt-4'\n", + " xml_response = search_arxiv(query)\n", + " \n", + " if xml_response:\n", + " # Parse the response and extract paper details\n", + " paper_details = parse_arxiv_response(xml_response)\n", + " \n", + " # Print the paper details\n", + " print(f\"Title: {paper_details['title']}\")\n", + " print(f\"Authors: {', '.join(paper_details['authors'])}\")\n", + " print(f\"Summary: {paper_details['summary']}\")\n", + " print(f\"Published Date: {paper_details['published_date']}\")\n", + " \n", + " # Analyze potential applications in software based on the summary\n", + " # This part requires human analysis and cannot be automated\n", + " print(\"\\nPotential applications in software:\")\n", + " print(\"The potential applications in software based on the summary of the paper would require a detailed analysis and understanding of the content. This step should be performed by a domain expert who can interpret the findings and suggest practical applications.\")\n", + "\n", + "if __name__ == '__main__':\n", + " main()\n", "```\n", "\n", - "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", + "This script will print the details of the latest paper about GPT-4 on arXiv. After running the script, you should manually analyze the summary to evaluate the potential applications in software. This analysis requires domain expertise and cannot be fully automated.\n", + "\n", + "Please run the script, and I will assist you with the analysis of the potential applications in software based on the information retrieved.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001B[0m\n" ] }, { @@ -266,103 +328,67 @@ "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Title: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", - "Authors: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", - "Abstract: In the field of document understanding, significant advances have been made\n", - "in the fine-tuning of Multimodal Large Language Models (MLLMs) with\n", - "instruction-following data. Nevertheless, the potential of text-grounding\n", - "capability within text-rich scenarios remains underexplored. In this paper, we\n", - "present a text-grounding document understanding model, termed TGDoc, which\n", - "addresses this deficiency by enhancing MLLMs with the ability to discern the\n", - "spatial positioning of text within images. Empirical evidence suggests that\n", - "text-grounding improves the model's interpretation of textual content, thereby\n", - "elevating its proficiency in comprehending text-rich images. Specifically, we\n", - "compile a dataset containing 99K PowerPoint presentations sourced from the\n", - "internet. We formulate instruction tuning tasks including text detection,\n", - "recognition, and spotting to facilitate the cohesive alignment between the\n", - "visual encoder and large language model. Moreover, we curate a collection of\n", - "text-rich images and prompt the text-only GPT-4 to generate 12K high-quality\n", - "conversations, featuring textual locations within text-rich scenarios. By\n", - "integrating text location data into the instructions, TGDoc is adept at\n", - "discerning text locations during the visual question process. Extensive\n", - "experiments demonstrate that our method achieves state-of-the-art performance\n", - "across multiple text-rich benchmarks, validating the effectiveness of our\n", - "method.\n", - "Publication Date: 2023-11-22 06:46:37+00:00\n", - "URL: http://arxiv.org/abs/2311.13194v1\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "\n", - "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", - "\n", - "**Abstract Summary and Potential Applications in Software Development:**\n", - "The paper discusses the enhancement of Multimodal Large Language Models (MLLMs) with text-grounding capabilities to improve document understanding. The authors present a model called TGDoc, which is designed to discern the spatial positioning of text within images. This is particularly useful for understanding text-rich images, such as PowerPoint presentations.\n", - "\n", - "The potential applications in software development could include:\n", - "\n", - "1. **Document Management Systems**: The TGDoc model could be integrated into document management systems to improve the searchability and categorization of documents by understanding the spatial layout of text within images.\n", + "Requirement already satisfied: requests in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (1.26.16)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (2023.7.22)\n", "\n", - "2. **Content Creation Tools**: Software tools for creating presentations or other visual content could benefit from TGDoc by providing suggestions or automations based on the textual content within the images.\n", + "Title: MEDITRON-70B: Scaling Medical Pretraining for Large Language Models\n", + "Authors: Zeming Chen, Alejandro Hernández Cano, Angelika Romanou, Antoine Bonnet, Kyle Matoba, Francesco Salvi, Matteo Pagliardini, Simin Fan, Andreas Köpf, Amirkeivan Mohtashami, Alexandre Sallinen, Alireza Sakhaeirad, Vinitra Swamy, Igor Krawczuk, Deniz Bayazit, Axel Marmet, Syrielle Montariol, Mary-Anne Hartley, Martin Jaggi, Antoine Bosselut\n", + "Summary: Large language models (LLMs) can potentially democratize access to medical\n", + "knowledge. While many efforts have been made to harness and improve LLMs'\n", + "medical knowledge and reasoning capacities, the resulting models are either\n", + "closed-source (e.g., PaLM, GPT-4) or limited in scale (<= 13B parameters),\n", + "which restricts their abilities. In this work, we improve access to large-scale\n", + "medical LLMs by releasing MEDITRON: a suite of open-source LLMs with 7B and 70B\n", + "parameters adapted to the medical domain. MEDITRON builds on Llama-2 (through\n", + "our adaptation of Nvidia's Megatron-LM distributed trainer), and extends\n", + "pretraining on a comprehensively curated medical corpus, including selected\n", + "PubMed articles, abstracts, and internationally-recognized medical guidelines.\n", + "Evaluations using four major medical benchmarks show significant performance\n", + "gains over several state-of-the-art baselines before and after task-specific\n", + "finetuning. Overall, MEDITRON achieves a 6% absolute performance gain over the\n", + "best public baseline in its parameter class and 3% over the strongest baseline\n", + "we finetuned from Llama-2. Compared to closed-source LLMs, MEDITRON-70B\n", + "outperforms GPT-3.5 and Med-PaLM and is within 5% of GPT-4 and 10% of\n", + "Med-PaLM-2. We release our code for curating the medical pretraining corpus and\n", + "the MEDITRON model weights to drive open-source development of more capable\n", + "medical LLMs.\n", + "Published Date: 2023-11-27T18:49:43Z\n", "\n", - "3. **Accessibility Features**: The ability to understand text within images could enhance accessibility features in software, allowing for better text-to-speech conversion for visually impaired users.\n", + "Potential applications in software:\n", + "The potential applications in software based on the summary of the paper would require a detailed analysis and understanding of the content. This step should be performed by a domain expert who can interpret the findings and suggest practical applications.\n", "\n", - "4. **Data Extraction and Analysis**: TGDoc could be used in data extraction tools to pull out relevant information from text-rich images for analysis or reporting purposes.\n", - "\n", - "5. **Enhanced User Interfaces**: User interfaces that can interpret the spatial layout of text could provide more intuitive interactions, especially in applications that deal with document editing or creation.\n", - "\n", - "6. **Automated Customer Support**: By understanding text within images, customer support bots could provide more accurate responses to user inquiries that involve screenshots or other visual aids.\n", - "\n", - "The paper's approach to integrating text location data into instructions for better comprehension of text-rich images could be a significant step forward in the development of more intelligent and context-aware software applications.\n", - "\n", - "**Publication Details:**\n", - "- **Title**: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", - "- **Authors**: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", - "- **Abstract URL**: [http://arxiv.org/abs/2311.13194v1](http://arxiv.org/abs/2311.13194v1)\n", - "- **Publication Date**: November 22, 2023\n", - "\n", - "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001B[33mDomain_expert\u001B[0m (to chat_manager):\n", "\n", + "Based on the summary of the paper titled \"MEDITRON-70B: Scaling Medical Pretraining for Large Language Models,\" we can infer several potential applications in software, particularly within the medical domain:\n", "\n", + "1. **Medical Knowledge Access**: MEDITRON-70B, as an open-source large language model (LLM) adapted to the medical domain, can be integrated into software platforms to provide healthcare professionals and researchers with easier access to medical knowledge. This could be in the form of intelligent search engines or decision support systems that can understand and process complex medical queries.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "2. **Clinical Decision Support**: The model's ability to understand and process medical literature can be leveraged to develop clinical decision support tools that assist healthcare providers in diagnosing and treating patients. By integrating MEDITRON-70B into electronic health record (EHR) systems, it could provide real-time, evidence-based recommendations.\n", "\n", + "3. **Medical Education**: Educational software can utilize MEDITRON-70B to create interactive learning experiences for medical students and professionals. The model can generate quizzes, explain complex medical concepts, and simulate patient interactions for training purposes.\n", "\n", + "4. **Research Acceleration**: Researchers can use MEDITRON-70B to quickly sift through vast amounts of medical literature to find relevant studies, thus accelerating the pace of medical research. This could be particularly useful in systematic reviews and meta-analyses.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", - "\n", - "\n", + "5. **Telemedicine**: In telemedicine applications, MEDITRON-70B could be used to pre-assess patient symptoms and provide preliminary advice, helping to triage cases and manage healthcare resources more effectively.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "6. **Language Translation and Localization**: Given the model's large-scale pretraining on medical texts, it could be used to translate medical documents and guidelines into different languages, making medical knowledge more accessible globally.\n", "\n", + "7. **Personalized Medicine**: Software applications could use MEDITRON-70B to analyze patient data and generate personalized treatment plans based on the latest medical research and guidelines.\n", "\n", + "8. **Healthcare Chatbots**: The model could power chatbots that provide medical information to patients, answer frequently asked questions, and guide them through administrative processes like appointment scheduling.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "9. **Drug Discovery and Development**: MEDITRON-70B could assist in the drug discovery process by analyzing scientific literature to identify potential drug targets and predict the effects of drug compounds.\n", "\n", + "10. **Public Health Monitoring**: The model could be used to monitor public health trends by analyzing social media, news articles, and other text sources for mentions of disease outbreaks or health-related events.\n", "\n", + "It's important to note that while these applications are promising, they also require careful consideration of ethical implications, data privacy, and the need for rigorous validation to ensure patient safety and the accuracy of medical information provided by the software.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", - "\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", - "\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", - "\n", - "\n", + "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n" ] @@ -385,16 +411,24 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "7fb0bfff01dd1330", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:29:39.150365Z", - "start_time": "2023-11-25T19:29:39.147791300Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:23:28.687772900Z", + "start_time": "2023-11-28T09:23:28.681728600Z" + } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "All agents have been cleared.\n" + ] + } + ], "source": [ "builder.clear_all_agents(recycle_endpoint=True)" ] @@ -441,14 +475,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "e4b88a5d482ceba4", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:41:44.198099300Z", - "start_time": "2023-11-25T19:41:44.194085500Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:23:29.950261800Z", + "start_time": "2023-11-28T09:23:29.944238100Z" + } }, "outputs": [ { @@ -475,70 +509,100 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "34addd498e5ab174", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:41:48.044519600Z", - "start_time": "2023-11-25T19:41:45.661666200Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:23:39.363905700Z", + "start_time": "2023-11-28T09:23:31.493375Z" + } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "Loding config from ./save_config_eb1be857faa608aeb4c5af11fe4ab245.json\n", + "Creating agent Data_scientist with backbone gpt-4-1106-preview...\n", + "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", + "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", + "Adding user console proxy...\n", "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", - "\n", - "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "Find a latest paper about Llava on arxiv and find its potential applications in multi-modal model.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", + "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", "\n", - "To accomplish this task, we will follow these steps:\n", + "To find the latest paper about \"Llava\" on arXiv and evaluate its potential applications in multi-modal models, we will break down the task into the following steps:\n", "\n", - "1. **Search arXiv for the latest papers on GPT-4**: We will use the `arxiv` Python library to search for the most recent papers related to GPT-4. If the `arxiv` library is not installed, you will need to install it using `pip install arxiv`.\n", + "1. Search for the latest papers on arXiv with the keyword \"Llava\".\n", + "2. Download and read the most recent paper found.\n", + "3. Analyze the content to determine its potential applications in multi-modal models.\n", "\n", - "2. **Extract Information**: From the search results, we will extract the title, authors, summary, and publication date of the latest paper.\n", + "Let's start with step 1. We will use the `arxiv` Python library to search for papers. If you don't have the `arxiv` library installed, you can install it using pip:\n", "\n", - "3. **Analyze for Potential Applications in Software Development**: We will read through the summary to identify mentions of potential applications in software development.\n", - "\n", - "4. **Present Findings**: We will print out the relevant information.\n", + "```bash\n", + "pip install arxiv\n", + "```\n", "\n", - "Let's start with step 1. Here is the Python code to search for the latest papers on GPT-4 on arXiv:\n", + "Now, let's write a Python script to search for the latest papers on arXiv with the keyword \"Llava\":\n", "\n", "```python\n", "import arxiv\n", + "import datetime\n", "\n", "# Define the search query and parameters\n", - "search_query = 'all:\"GPT-4\"'\n", + "search_query = 'all:Llava'\n", "sort_by = arxiv.SortCriterion.SubmittedDate\n", "sort_order = arxiv.SortOrder.Descending\n", "\n", - "# Perform the search\n", + "# Search for papers on arXiv\n", "search = arxiv.Search(\n", - " query=search_query,\n", - " max_results=1,\n", - " sort_by=sort_by,\n", - " sort_order=sort_order\n", + " query=search_query,\n", + " max_results=1,\n", + " sort_by=sort_by,\n", + " sort_order=sort_order\n", ")\n", "\n", "# Fetch the result\n", - "for result in search.results():\n", - " print(\"Title:\", result.title)\n", - " print(\"Authors:\", \", \".join(author.name for author in result.authors))\n", - " print(\"Abstract:\", result.summary)\n", - " print(\"Publication Date:\", result.published)\n", - " print(\"URL:\", result.entry_id)\n", - " break # Since we only want the latest paper, we break after the first result\n", + "try:\n", + " paper = next(search.results())\n", + " print(f\"Title: {paper.title}\")\n", + " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", + " print(f\"Abstract: {paper.summary}\")\n", + " print(f\"Published: {paper.published}\")\n", + " print(f\"URL: {paper.entry_id}\")\n", + "except StopIteration:\n", + " print(\"No papers found with the keyword 'Llava'.\")\n", + "except Exception as e:\n", + " print(f\"An error occurred: {e}\")\n", "```\n", "\n", - "Please run this code in your Python environment to retrieve the latest paper on GPT-4 from arXiv. Once you have the paper, we can proceed to analyze its content for potential applications in software development.\n", + "This script will search for the most recent papers on arXiv with the keyword \"Llava\" and print out the title, authors, abstract, publication date, and URL of the latest paper.\n", + "\n", + "Please run this script in your Python environment to find the latest paper. Once we have the paper, we can proceed to step 2 and 3 to download, read, and analyze its content for potential applications in multi-modal models.\n", + "\n", + "After running the script, please provide the title and abstract of the paper so that we can discuss its potential applications.\n", "\n", "--------------------------------------------------------------------------------\n", "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001B[0m\n" ] }, { @@ -556,110 +620,77 @@ "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Title: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", - "Authors: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", - "Abstract: In the field of document understanding, significant advances have been made\n", - "in the fine-tuning of Multimodal Large Language Models (MLLMs) with\n", - "instruction-following data. Nevertheless, the potential of text-grounding\n", - "capability within text-rich scenarios remains underexplored. In this paper, we\n", - "present a text-grounding document understanding model, termed TGDoc, which\n", - "addresses this deficiency by enhancing MLLMs with the ability to discern the\n", - "spatial positioning of text within images. Empirical evidence suggests that\n", - "text-grounding improves the model's interpretation of textual content, thereby\n", - "elevating its proficiency in comprehending text-rich images. Specifically, we\n", - "compile a dataset containing 99K PowerPoint presentations sourced from the\n", - "internet. We formulate instruction tuning tasks including text detection,\n", - "recognition, and spotting to facilitate the cohesive alignment between the\n", - "visual encoder and large language model. Moreover, we curate a collection of\n", - "text-rich images and prompt the text-only GPT-4 to generate 12K high-quality\n", - "conversations, featuring textual locations within text-rich scenarios. By\n", - "integrating text location data into the instructions, TGDoc is adept at\n", - "discerning text locations during the visual question process. Extensive\n", - "experiments demonstrate that our method achieves state-of-the-art performance\n", - "across multiple text-rich benchmarks, validating the effectiveness of our\n", - "method.\n", - "Publication Date: 2023-11-22 06:46:37+00:00\n", - "URL: http://arxiv.org/abs/2311.13194v1\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "\n", - "Based on the information retrieved from arXiv, the latest paper related to GPT-4 is titled \"Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\" by Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, and Houqiang Li. The paper was published on November 22, 2023.\n", - "\n", - "**Abstract Summary and Potential Applications in Software Development:**\n", - "The paper discusses the enhancement of Multimodal Large Language Models (MLLMs) with text-grounding capabilities to improve document understanding. The authors present a model called TGDoc, which is designed to discern the spatial positioning of text within images. This is particularly useful for understanding text-rich images, such as PowerPoint presentations.\n", - "\n", - "The potential applications in software development could include:\n", - "\n", - "1. **Document Management Systems**: The TGDoc model could be integrated into document management systems to improve the searchability and categorization of documents by understanding the spatial layout of text within images.\n", - "\n", - "2. **Content Creation Tools**: Software tools for creating presentations or other visual content could benefit from TGDoc by providing suggestions or automations based on the textual content within the images.\n", - "\n", - "3. **Accessibility Features**: The ability to understand text within images could enhance accessibility features in software, allowing for better text-to-speech conversion for visually impaired users.\n", - "\n", - "4. **Data Extraction and Analysis**: TGDoc could be used in data extraction tools to pull out relevant information from text-rich images for analysis or reporting purposes.\n", - "\n", - "5. **Enhanced User Interfaces**: User interfaces that can interpret the spatial layout of text could provide more intuitive interactions, especially in applications that deal with document editing or creation.\n", - "\n", - "6. **Automated Customer Support**: By understanding text within images, customer support bots could provide more accurate responses to user inquiries that involve screenshots or other visual aids.\n", - "\n", - "The paper's approach to integrating text location data into instructions for better comprehension of text-rich images could be a significant step forward in the development of more intelligent and context-aware software applications.\n", - "\n", - "**Publication Details:**\n", - "- **Title**: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", - "- **Authors**: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", - "- **Abstract URL**: [http://arxiv.org/abs/2311.13194v1](http://arxiv.org/abs/2311.13194v1)\n", - "- **Publication Date**: November 22, 2023\n", - "\n", - "TERMINATE\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "Requirement already satisfied: arxiv in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.0.0)\n", + "Requirement already satisfied: feedparser==6.0.10 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from arxiv) (6.0.10)\n", + "Requirement already satisfied: requests==2.31.0 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from arxiv) (2.31.0)\n", + "Requirement already satisfied: sgmllib3k in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from feedparser==6.0.10->arxiv) (1.0.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (1.26.16)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (2023.7.22)\n", "\n", + "Title: PG-Video-LLaVA: Pixel Grounding Large Video-Language Models\n", + "Authors: Shehan Munasinghe, Rusiru Thushara, Muhammad Maaz, Hanoona Abdul Rasheed, Salman Khan, Mubarak Shah, Fahad Khan\n", + "Abstract: Extending image-based Large Multimodal Models (LMM) to videos is challenging\n", + "due to the inherent complexity of video data. The recent approaches extending\n", + "image-based LMM to videos either lack the grounding capabilities (e.g.,\n", + "VideoChat, Video-ChatGPT, Video-LLaMA) or do not utilize the audio-signals for\n", + "better video understanding (e.g., Video-ChatGPT). Addressing these gaps, we\n", + "propose Video-LLaVA, the first LMM with pixel-level grounding capability,\n", + "integrating audio cues by transcribing them into text to enrich video-context\n", + "understanding. Our framework uses an off-the-shelf tracker and a novel\n", + "grounding module, enabling it to spatially and temporally localize objects in\n", + "videos following user instructions. We evaluate Video-LLaVA using video-based\n", + "generative and question-answering benchmarks and introduce new benchmarks\n", + "specifically designed to measure prompt-based object grounding performance in\n", + "videos. Further, we propose the use of Vicuna over GPT-3.5, as utilized in\n", + "Video-ChatGPT, for video-based conversation benchmarking, ensuring\n", + "reproducibility of results which is a concern with the proprietary nature of\n", + "GPT-3.5. Our framework builds on SoTA image-based LLaVA model and extends its\n", + "advantages to the video domain, delivering promising gains on video-based\n", + "conversation and grounding tasks. Project Page:\n", + "https://github.com/mbzuai-oryx/Video-LLaVA\n", + "Published: 2023-11-22 14:48:30+00:00\n", + "URL: http://arxiv.org/abs/2311.13435v1\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001B[33mDomain_expert\u001B[0m (to chat_manager):\n", "\n", + "Based on the information provided, the paper titled \"PG-Video-LLaVA: Pixel Grounding Large Video-Language Models\" presents a Large Multimodal Model (LMM) that extends image-based LMMs to videos by incorporating pixel-level grounding capabilities and integrating audio cues through text transcription to enhance video-context understanding.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "The potential applications of Video-LLaVA in multi-modal models are significant:\n", "\n", + "1. **Enhanced Video Understanding**: Video-LLaVA's ability to spatially and temporally localize objects in videos can be applied to improve video analysis systems, such as those used in surveillance, sports analytics, and content moderation.\n", "\n", + "2. **Interactive Video Systems**: The grounding module allows for interactive systems where users can query specific objects or events within a video, which could be useful in educational platforms, interactive storytelling, or customer service bots that interact with video content.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "3. **Accessibility**: By transcribing audio signals and grounding visual content, Video-LLaVA can aid in creating more accessible content for individuals with hearing or visual impairments, providing richer descriptions of video content.\n", "\n", + "4. **Content Creation and Editing**: The model's grounding capabilities can be leveraged in video editing software to allow creators to manipulate video content through natural language commands, streamlining the editing process.\n", "\n", + "5. **Video-Based Conversational Agents**: The paper mentions video-based conversation benchmarking, which implies that Video-LLaVA could be used to develop conversational agents that understand and respond to video content, enhancing customer support or entertainment applications.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "6. **Research and Education**: In academic settings, Video-LLaVA could be used to analyze educational videos, providing summaries, answering questions, and helping students locate information within video lectures.\n", "\n", + "7. **Marketing and Advertising**: Companies could use Video-LLaVA to analyze consumer videos for product placement, sentiment analysis, and to understand consumer behavior through video content.\n", "\n", + "8. **Integration with Robotics**: Robots equipped with video processing capabilities could use Video-LLaVA to better understand their environment and interact with it based on visual and audio cues.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", - "\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "The paper's introduction of new benchmarks for prompt-based object grounding performance in videos also suggests that Video-LLaVA could push the boundaries of what is possible in video understanding and interaction, leading to more sophisticated multi-modal applications.\n", "\n", + "For further analysis, one would need to delve into the technical details of the paper, experiment with the model, and possibly contribute to the project, which is available on the provided GitHub page.\n", "\n", + "TERMINATE\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", - "\n", - "It seems there might be some confusion or an issue with the user console. If you have any further questions or need assistance with another task, please let me know, and I'll be happy to help. If you're done, no further action is needed. Thank you!\n", - "\n", - "--------------------------------------------------------------------------------\n" + "All agents have been cleared.\n" ] } ], "source": [ "new_builder = AgentBuilder(config_path=config_path).load(saved_path)\n", - "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", + "new_builder.start(task=\"Find a latest paper about Llava on arxiv and find its potential applications in multi-modal model.\")\n", "new_builder.clear_all_agents()" ] }, @@ -678,118 +709,106 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "id": "4051c25b2cd1918c", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-25T19:43:09.367458800Z", - "start_time": "2023-11-25T19:42:15.653608Z" - }, - "collapsed": false + "end_time": "2023-11-28T09:26:01.069144900Z", + "start_time": "2023-11-28T09:23:39.355869200Z" + } }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "assistant_id was None, creating a new assistant\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ "Generating agents...\n", - "Data_scientist,Research_analyst,Software_developer are generated.\n", + "Data_scientist,Domain_expert,Software_engineer are generated.\n", "Preparing configuration for Data_scientist...\n", - "Preparing configuration for Research_analyst...\n", - "Preparing configuration for Software_developer...\n" + "Preparing configuration for Domain_expert...\n", + "Preparing configuration for Software_engineer...\n", + "Creating agent Data_scientist with backbone gpt-4-1106-preview...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "assistant_id was None, creating a new assistant\n", - "assistant_id was None, creating a new assistant\n" + "Multiple assistants with name Data_scientist found. Using the first assistant in the list. Please specify the assistant ID in llm_config to use a specific assistant.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ + "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", + "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", + "Adding user console proxy...\n", "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", - "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "Find a latest paper about XAI on arxiv and find its potential applications in medical.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_developer\u001B[0m (to chat_manager):\n", + "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", "\n", - "To accomplish the task of finding the latest paper about GPT-4 on arXiv and identifying its potential applications in software development, we will break down the process into the following steps:\n", + "To accomplish the task of finding the latest paper about Explainable Artificial Intelligence (XAI) on arXiv and evaluating its potential applications in the medical field, we will break down the task into several steps:\n", "\n", - "1. **Search for papers on arXiv**: We'll use the arXiv API to search for recent papers mentioning GPT-4. The arXiv API allows us to programmatically query their database.\n", + "1. **Search arXiv for the latest papers on XAI**: Use the arXiv API to search for papers that match the topic \"Explainable Artificial Intelligence\".\n", + "2. **Download and read the paper's summary**: Extract and present the content of the paper's summary to gain insight into its subject matter.\n", + "3. **Evaluate real-world applications in medicine**: Analyze the content for potential applications within the medical field.\n", "\n", - "2. **Retrieve and analyze the paper**: Once we have identified relevant papers, we will download the metadata of the latest paper. We will inspect the abstract and other available metadata for mentions of potential applications in software development.\n", + "Let's start with the first step:\n", "\n", - "3. **Present findings**: Finally, we will present a summary of our findings.\n", + "### Step 1 - Searching for the latest XAI papers on arXiv\n", "\n", - "Here is a Python script that uses the arXiv API to search for the latest paper mentioning GPT-4:\n", + "We will use the `arxiv` Python library to search for papers. If you don't have it installed, you can install it using `pip install arxiv`.\n", "\n", "```python\n", - "import urllib\n", - "import urllib.request\n", - "import feedparser\n", - "\n", - "# Define a function to search the arXiv API\n", - "def search_arxiv(query):\n", - " # Base URL for the arXiv API\n", - " base_url = 'http://export.arxiv.org/api/query?'\n", - " \n", - " # Parameters for the API query\n", - " params = {\n", - " 'search_query': query,\n", - " 'sortBy': 'submittedDate',\n", - " 'sortOrder': 'descending'\n", - " }\n", - " \n", - " # Encode the parameters and make a GET request to the arXiv API\n", - " query_string = urllib.parse.urlencode(params)\n", - " with urllib.request.urlopen(base_url + query_string) as response:\n", - " response_text = response.read()\n", - " \n", - " # Parse the response using feedparser\n", - " feed = feedparser.parse(response_text)\n", - " return feed.entries\n", + "import arxiv\n", + "import datetime\n", "\n", - "# Search for papers on GPT-4\n", - "papers = search_arxiv('all:\"GPT-4\"')\n", + "# Define the query parameters\n", + "query = \"cat:cs.AI AND ti:XAI\"\n", + "sort_by = arxiv.SortCriterion.SubmittedDate # Sort by most recent submissions\n", "\n", - "if papers:\n", - " # Get the latest paper\n", - " latest_paper = papers[0]\n", - " \n", - " # Check for discussions of software development applications in the paper's summary\n", - " summary_lower = latest_paper.summary.lower()\n", - " software_related_terms = ['software', 'developing', 'programming', 'coding']\n", - " software_applications = any(term in summary_lower for term in software_related_terms)\n", - " \n", - " # Print out the details of the latest paper\n", - " print(f\"Title: {latest_paper.title}\")\n", - " print(f\"Authors: {', '.join(author.name for author in latest_paper.authors)}\")\n", - " print(f\"Abstract: {latest_paper.summary}\")\n", - " print(f\"Published: {latest_paper.published}\")\n", - " print(f\"Link: {latest_paper.link}\")\n", - " \n", - " # Print out whether software applications were mentioned\n", - " if software_applications:\n", - " print(\"\\nThe paper discusses potential applications in software development.\")\n", - " else:\n", - " print(\"\\nThe paper does not specifically discuss software development applications.\")\n", - "else:\n", - " print(\"No papers found on GPT-4.\")\n", + "# Search for papers on arXiv that match the query\n", + "search = arxiv.Search(\n", + " query=query,\n", + " sort_by=sort_by,\n", + " sort_order=arxiv.SortOrder.Descending,\n", + " max_results=1\n", + ")\n", "\n", + "# Fetch the latest paper\n", + "for paper in search.results():\n", + " print(f\"Title: {paper.title}\")\n", + " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", + " print(f\"Submitted Date: {paper.published}\")\n", + " print(f\"Summary: {paper.summary.replace('\\n', ' ')}\")\n", + " print(f\"URL: {paper.entry_id}\")\n", + " break # We only want the latest paper\n", "```\n", "\n", - "Please run this script to search for and analyze the latest papers on GPT-4. The script will print out information about the most recent paper on this topic and if it finds mention of applications in software development in the abstract. If any errors occur or if the script does not produce the expected outcome, please let me know, and I will provide assistance.\n", + "Once you run this script, it should output the title, authors, submission date, and summary of the latest paper about Explainable Artificial Intelligence on arXiv, as well as provide a URL to the paper.\n", + "\n", + "### Step 2 - Reading the paper's summary\n", + "\n", + "The script above already takes care of downloading and presenting the summary of the latest paper. This will provide us with an overview of the paper's content and potential applications.\n", + "\n", + "### Step 3 - Evaluating potential medical applications\n", + "\n", + "For evaluating real-world applications in medicine, we would need to read and understand the content of the paper's summary. Especially, we would look for:\n", + "- Any mention of medical datasets or case studies within the paper summary.\n", + "- Generalizable findings or methods that could be applied to medical scenarios, such as diagnostic tools, drug discovery, patient data analysis, etc.\n", + "\n", + "After you have executed the Python script and obtained the latest paper's summary information, review the content to determine if there are potential applications in the medical field. I can guide you through this analysis by asking you to consider the following questions:\n", + "\n", + "- Does the paper discuss applications or case studies related to medicine or healthcare?\n", + "- Are the methods or approaches described in the paper suitable for tasks in the medical domain, such as interpreting medical images, analyzing genomic data, or personalizing treatment plans?\n", + "\n", + "If you share the summary with me, I can help in conceptualizing how the findings in the paper might be applied to the medical field. However, I cannot analyse the content myself, you will have to apply this analysis based on my guidance.\n", + "\n", + "Please run the code provided to find the latest paper on XAI from arXiv and share the summary with me for further guidance on potential medical applications.\n", "\n", "\n", "--------------------------------------------------------------------------------\n", @@ -810,97 +829,108 @@ "text": [ "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", - "exitcode: 0 (execution succeeded)\n", + "exitcode: 1 (execution failed)\n", "Code output: \n", - "Title: Towards Improving Document Understanding: An Exploration on\n", - " Text-Grounding via MLLMs\n", - "Authors: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", - "Abstract: In the field of document understanding, significant advances have been made\n", - "in the fine-tuning of Multimodal Large Language Models (MLLMs) with\n", - "instruction-following data. Nevertheless, the potential of text-grounding\n", - "capability within text-rich scenarios remains underexplored. In this paper, we\n", - "present a text-grounding document understanding model, termed TGDoc, which\n", - "addresses this deficiency by enhancing MLLMs with the ability to discern the\n", - "spatial positioning of text within images. Empirical evidence suggests that\n", - "text-grounding improves the model's interpretation of textual content, thereby\n", - "elevating its proficiency in comprehending text-rich images. Specifically, we\n", - "compile a dataset containing 99K PowerPoint presentations sourced from the\n", - "internet. We formulate instruction tuning tasks including text detection,\n", - "recognition, and spotting to facilitate the cohesive alignment between the\n", - "visual encoder and large language model. Moreover, we curate a collection of\n", - "text-rich images and prompt the text-only GPT-4 to generate 12K high-quality\n", - "conversations, featuring textual locations within text-rich scenarios. By\n", - "integrating text location data into the instructions, TGDoc is adept at\n", - "discerning text locations during the visual question process. Extensive\n", - "experiments demonstrate that our method achieves state-of-the-art performance\n", - "across multiple text-rich benchmarks, validating the effectiveness of our\n", - "method.\n", - "Published: 2023-11-22T06:46:37Z\n", - "Link: http://arxiv.org/abs/2311.13194v1\n", - "\n", - "The paper does not specifically discuss software development applications.\n", + " File \"\", line 21\n", + " print(f\"Summary: {paper.summary.replace('\\n', ' ')}\")\n", + " ^\n", + "SyntaxError: f-string expression part cannot include a backslash\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", - "It appears that the script successfully retrieved information about the latest paper related to GPT-4:\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", "\n", - "- **Title**: Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs\n", - "- **Authors**: Yonghui Wang, Wengang Zhou, Hao Feng, Keyi Zhou, Houqiang Li\n", - "- **Abstract**: The paper presents a text-grounding document understanding model, termed TGDoc, which enhances Multimodal Large Language Models (MLLMs) with the ability to discern the spatial positioning of text within images. While the paper involves text-only GPT-4 to generate high-quality conversations within text-rich scenarios, it is focused on document understanding and text-grounding, not directly on software development applications.\n", - "- **Published Date**: 2023-11-22\n", - "- **Link**: [Towards Improving Document Understanding: An Exploration on Text-Grounding via MLLMs](http://arxiv.org/abs/2311.13194v1)\n", - " \n", - "Based on the abstract, the paper does not specifically discuss applications in software development. However, there might be indirect applications or implications that are not detailed in the abstract, and further reading of the full paper would be necessary to uncover those.\n", + "It appears there was an error due to the use of a backslash within an f-string expression. Let's correct the script to properly print the summary without causing a syntax error. We can use double quotes for the string inside the `replace` method to avoid this issue.\n", "\n", - "Since the provided abstract does not directly mention applications in software development, I am unable to extract insights regarding such applications without further analysis of the full document, which goes beyond the scope of this interaction.\n", + "Here is the corrected Python script:\n", "\n", - "If needed, I can assist with the next steps or provide additional services within the constraints given.\n", + "```python\n", + "import arxiv\n", + "import datetime\n", "\n", - "TERMINATE\n", + "# Define the query parameters\n", + "query = \"cat:cs.AI AND ti:XAI\"\n", + "sort_by = arxiv.SortCriterion.SubmittedDate # Sort by most recent submissions\n", "\n", + "# Search for papers on arXiv that match the query\n", + "search = arxiv.Search(\n", + " query=query,\n", + " sort_by=sort_by,\n", + " sort_order=arxiv.SortOrder.Descending,\n", + " max_results=1\n", + ")\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "# Fetch the latest paper\n", + "for paper in search.results():\n", + " print(f\"Title: {paper.title}\")\n", + " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", + " print(f\"Submitted Date: {paper.published}\")\n", + " summary = paper.summary.replace(\"\\n\", \" \")\n", + " print(f\"Summary: {summary}\")\n", + " print(f\"URL: {paper.entry_id}\")\n", + " break # We only want the latest paper\n", + "```\n", "\n", + "Please run the corrected script to get the details of the latest paper regarding Explainable Artificial Intelligence (XAI) on arXiv. Once you obtain the summary, we can discuss how to conceptualize its real-world applications in the medical field.\n", "\n", "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Title: Forms of Understanding of XAI-Explanations\n", + "Authors: Hendrik Buschmeier, Heike M. Buhl, Friederike Kern, Angela Grimminger, Helen Beierling, Josephine Fisher, André Groß, Ilona Horwath, Nils Klowait, Stefan Lazarov, Michael Lenke, Vivien Lohmer, Katharina Rohlfing, Ingrid Scharlau, Amit Singh, Lutz Terfloth, Anna-Lisa Vollmer, Yu Wang, Annedore Wilmes, Britta Wrede\n", + "Submitted Date: 2023-11-15 08:06:51+00:00\n", + "Summary: Explainability has become an important topic in computer science and artificial intelligence, leading to a subfield called Explainable Artificial Intelligence (XAI). The goal of providing or seeking explanations is to achieve (better) 'understanding' on the part of the explainee. However, what it means to 'understand' is still not clearly defined, and the concept itself is rarely the subject of scientific investigation. This conceptual article aims to present a model of forms of understanding in the context of XAI and beyond. From an interdisciplinary perspective bringing together computer science, linguistics, sociology, and psychology, a definition of understanding and its forms, assessment, and dynamics during the process of giving everyday explanations are explored. Two types of understanding are considered as possible outcomes of explanations, namely enabledness, 'knowing how' to do or decide something, and comprehension, 'knowing that' -- both in different degrees (from shallow to deep). Explanations regularly start with shallow understanding in a specific domain and can lead to deep comprehension and enabledness of the explanandum, which we see as a prerequisite for human users to gain agency. In this process, the increase of comprehension and enabledness are highly interdependent. Against the background of this systematization, special challenges of understanding in XAI are discussed.\n", + "URL: http://arxiv.org/abs/2311.08760v1\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001B[33mDomain_expert\u001B[0m (to chat_manager):\n", "\n", + "Based on the summary of the paper titled \"Forms of Understanding of XAI-Explanations,\" we can glean that this paper is conceptual in nature, aiming to define and explore the concept of 'understanding' in the context of Explainable Artificial Intelligence (XAI). The interdisciplinary perspective adopted by the authors examines the dynamics of understanding during the process of providing explanations, particularly focusing on two types of understanding: enabledness (knowing how) and comprehension (knowing that), ranging from shallow to deep levels.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "While the paper does not explicitly target the medical field, we can conceptualize its potential applications in medicine by considering how these concepts could enhance AI's role in healthcare:\n", "\n", + "1. **Patient Education**: Utilizing the concept of 'understanding' to explain AI-driven medical diagnoses or treatment plans to patients, enabling them to comprehend their condition and the suggested medical interventions. By converting complex medical data into understandable explanations, patients can make more informed decisions about their health.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "2. **Clinical Decision Support**: Improvement in clinician's adoption of AI tools for decision-making by employing explanations that focus on enabledness (knowing how) to use AI recommendations and comprehension (knowing that) to understand the underlying reasons for AI's advice.\n", "\n", + "3. **Medical Study and Research**: Enabling medical researchers to leverage AI tools that can explain their outputs in a way that provides deep understanding, thereby increasing trust in AI's ability to identify patterns and offer insights in complex datasets like genomic information or epidemiological studies.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "4. **Training Healthcare Professionals**: Enhancing the educational tools for healthcare professionals by incorporating XAI systems that provide varying degrees of understanding, from basic conceptual learning to advanced clinical applications.\n", "\n", + "5. **Ethics and Legal Compliance**: The ability to provide understandable explanations for AI's actions could facilitate compliance with ethical guidelines and legal requirements, especially in sensitive areas such as patient data handling or algorithmic decisions affecting patient care.\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "In summary, while the paper presents a conceptual model of understanding applicable to XAI systems, its principles could potentially be adapted and used to develop AI tools that are more transparent, trusted, and user-friendly in the medical domain. Applications could span patient interaction, clinical support, medical research, educational initiatives, and compliance requirements, all of which benefit from a higher degree of explainability and understanding provided by AI systems.\n", "\n", + "TERMINATE\n", "\n", "\n", - "--------------------------------------------------------------------------------\n" + "--------------------------------------------------------------------------------\n", + "All agents have been cleared.\n" ] } ], "source": [ "new_builder = AgentBuilder(config_path=config_path)\n", "new_builder.build(building_task, default_llm_config, use_gpts=True) # Transfer to GPTs API.\n", - "new_builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")\n", + "new_builder.start(task=\"Find a latest paper about XAI on arxiv and find its potential applications in medical.\")\n", "new_builder.clear_all_agents()" ] }, diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py new file mode 100644 index 00000000000..e34104cfa30 --- /dev/null +++ b/test/agentchat/contrib/test_autobuder.py @@ -0,0 +1,97 @@ +import pytest +import os +import json +from autogen.agentchat.contrib.agent_builder import AgentBuilder + + +oai_config_path = "OAI_CONFIG_LIST" + + +def test_build(): + builder = AgentBuilder(config_path=oai_config_path) + building_task = ( + "Find a paper on arxiv by programming, and analysis its application in some domain. " + "For example, find a latest paper about gpt-4 on arxiv " + "and find its potential applications in software." + ) + + builder.build(building_task=building_task, default_llm_config={"temperature": 0}) + + # check number of agents + assert len(builder.agent_procs_assign.keys()) <= builder.max_agents + + # check system message + for agent, proc in builder.agent_procs_assign.values(): + assert "TERMINATE" in agent.system_message + + +def test_save(): + builder = AgentBuilder(config_path=oai_config_path) + building_task = ( + "Find a paper on arxiv by programming, and analysis its application in some domain. " + "For example, find a latest paper about gpt-4 on arxiv " + "and find its potential applications in software." + ) + + builder.build(building_task=building_task, default_llm_config={"temperature": 0}) + saved_files = builder.save("../../test_files/save_config_test.json") + + # check config file path + assert os.path.isfile(saved_files) + + saved_configs = json.load(open(saved_files)) + + # check config format + assert saved_configs.get("building_task", None) is not None + assert saved_configs.get("agent_configs", None) is not None + assert saved_configs.get("manager_system_message", None) is not None + assert saved_configs.get("coding", None) is not None + assert saved_configs.get("default_llm_config", None) is not None + + +def test_load(): + builder = AgentBuilder(config_path=oai_config_path) + + config_save_path = "../../test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json" + configs = json.load(open(config_save_path)) + agent_configs = { + e["name"]: {"model": e["model"], "system_message": e["system_message"]} for e in configs["agent_configs"] + } + + builder.load(config_save_path) + + # check config loading + assert builder.coding == configs["coding"] + for agent in builder.agent_procs_assign.values(): + agent_name = agent[0].name + assert agent_configs.get(agent_name, None) is not None + assert agent_configs[agent_name]["model"] == agent[0].llm_config["model"] + assert agent_configs[agent_name]["system_message"] == agent[0].system_message + + +def test_clear_agent(): + builder = AgentBuilder(config_path=oai_config_path) + + config_save_path = "../../test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json" + builder.load(config_save_path) + builder.clear_all_agents() + + # check if the agent cleared + assert len(builder.agent_procs_assign) == 0 + + +def test_start(): + builder = AgentBuilder(config_path=oai_config_path) + config_save_path = "../../test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json" + builder.load(config_save_path) + test_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." + + group_chat, _ = builder.start(task=test_task) + history = group_chat.messages.copy() + + assert history[0]["content"] == test_task + history.reverse() + for msg in history: + if msg["content"] != "": + assert "TERMINATE" in msg["content"] + break diff --git a/test/test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json b/test/test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json new file mode 100644 index 00000000000..38e5e903122 --- /dev/null +++ b/test/test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json @@ -0,0 +1,25 @@ +{ + "building_task": "Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.", + "agent_configs": [ + { + "name": "Data_scientist", + "model": "gpt-4-1106-preview", + "system_message": "As a Data Scientist, you will:\n\n- Utilize your advanced coding skills specifically in Python to automate information gathering from various sources including web scraping, file downloads, and parsing data. This may include writing Python scripts to retrieve and present the latest research papers from preprint services like arXiv.\n- Apply your analytical acumen to conduct thorough examinations of the technical materials you gather, especially focusing on their practical applications within different domains, such as software development in the case of GPT-4 research papers.\n- Perform data processing tasks that may involve complex algorithmic work, statistical analysis, or machine learning methodologies to extract insights and build models based on the gathered information, executing Python code as necessary to accomplish these tasks.\n- Present findings with clarity, extracting and interpreting results solely from the execution of Python scripts you've crafted. Use 'print' functions adequately in your Python code to ensure all results are clear and interpretable.\n- Be diligent in checking the viability and correctness of your code and analysis. When errors occur, address them promptly and provide corrected Python code for execution.\n- Remain adaptive to the dynamic field of data science, continually seeking additional relevant information when required, and revising your approach to problem-solving as needed.\n- Persistently strive for the successful completion of the task at hand, ready to pursue alternative strategies in case initial methods fall short of fulfilling the task's requirements.\n- Conclude any sequence of task-related interactions with a final confirmation that the user's needs have been met, signifying the end of the process by replying \"TERMINATE\"." + }, + { + "name": "Domain_expert", + "model": "gpt-4-1106-preview", + "system_message": "As a Domain Expert, you leverage your deep understanding and analytical abilities to provide insights and applications of new findings in scholarly articles. Your role focuses on identifying, interpreting, and discussing the implications of cutting-edge research in a specific domain. You will:\n\n1. Employ Python programming to autonomously locate and retrieve academic papers from databases such as arXiv. This involves formulating queries, processing search results, and downloading relevant documents using automated scripts.\n\n2. Analyze and synthesize the information contained within the located papers, with a particular emphasis on assessing their applications in the specified domain. Your language skills will be pivotal in understanding complex scientific texts and elucidating their potential impact on real-world problems and industry practices.\n\n3. Clearly communicate your findings and developed applications, providing comprehensive insights into how the content of the research paper can be utilized or integrated into existing systems or processes within your domain of expertise.\n\n4. Your work will be structured and systematic, starting from the initial programming stage to the final analysis and communication. Each phase should be clearly demarcated, with an explanation of your methodology and steps taken.\n\n5. Ensure all coding is provided in Python, and your guidance will be executed directly without the need for user modifications or intervention beyond the execution of provided scripts.\n\n6. You will manage any encountered issues during the process, including correcting errors in code and revising your approach based on the results obtained from script execution.\n\n7. Upon completing your task and providing a thorough analysis, confirm your final output and conclude the interaction with the statement \"TERMINATE,\" signaling the successful satisfaction of the user's need." + }, + { + "name": "Software_engineer", + "model": "gpt-4-1106-preview", + "system_message": "As a skilled Software Engineer, your primary role is to leverage your coding expertise, particularly in Python, to facilitate the discovery and analysis of academic papers on arXiv, and to evaluate their real-world applications. \n\n1. You are expected to craft Python scripts capable of web tasks such as searching for academic papers, downloading and reading files, extracting and presenting content, as well as recognizing the current date/time and operating system details. Your script should output all necessary information for task completion.\n\n2. You should use Python scripts to accomplish specific tasks, ensuring that the script completes the task autonomously and provides the results to the user.\n\nYour responsibilities involve executing tasks in a systematic manner, clarifying your approach when a plan is not provided. Clearly distinguish between steps that involve executing Python code and those that engage your analytical skills. \n\nAlways present your Python code within a code block, ensuring it is ready for immediate execution without requiring modifications from the user. Here is how you should format a code suggestion:\n```python\n# Python code goes here\n```\n\nIf a script is to be saved before execution, indicate the filename at the beginning of the code block. Do not include multiple code blocks in a single interaction or ask users to manually copy results \u2014 use the `print` function within the script to display outputs. After providing a script, review the user's execution result. In case of an error, deliver a corrected script. If the task remains unsolved despite error-free execution, reassess your approach, gather more information if needed, and try a different strategy.\n\nEnsure that your solution is methodically verified and, where possible, supported by verifiable evidence.\n\nConclude your interaction by replying \u201cTERMINATE\u201d once the task is complete and the user\u2019s need has been satisfied. \n\nRemember, while your role is to assist with a task, it is also to enable and educate, ultimately fostering a user's understanding and their ability to independently solve similar problems in the future." + } + ], + "manager_system_message": "Group chat manager.", + "coding": true, + "default_llm_config": { + "temperature": 0 + } +} From 08df3149e6260a1e6e42df91cf3b73b2340a29b9 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 03:44:38 +0900 Subject: [PATCH 42/74] modify blog for test --- .../blog/2023-11-26-Agent-AutoBuild/index.mdx | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index 94d9a6a944f..02ff6775552 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -3,11 +3,28 @@ title: Agent AutoBuild - Automatically Building Multi-agent Systems authors: - LinxinS97 - jieyuz2 -tags: [LLM] +tags: [LLM, research] --- ![system](img/system.png) **TL;DR:** +Introducing **AutoBuild**, building multi-agent system automatically, fast, and easily for complex tasks with minimal +user prompt required, powered by a new designed class **AgentBuilder**. AgentBuilder also support open-source LLMs by +leveraging [vLLM](https://docs.vllm.ai/en/latest/index.html) and [FastChat](https://github.com/lm-sys/FastChat). +Checkout example notebooks and file for reference: +- [AutoBuild Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_autobuild.ipynb) +- [AgentBuilder](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/agent_builder.py) +## Introduction +In this blog, we introduce **AutoBuild**, a pipeline that can automatically build multi-agent system for complex task. +Specifically, we design a new class called **AgentBuilder**, which will complete the generation of participant expert agents +and the construction of group chat automatically after the user provide descriptions of a building task and a execution task. + +AgentBuilder support open-source models on Hugging Face powered by [vLLM](https://docs.vllm.ai/en/latest/index.html) +and [FastChat](https://github.com/lm-sys/FastChat). Once the user choose to use open-source LLM, AgentBuilder will set +up an endpoint server automatically without any user participant. + +### Pipeline +The pipeline of AutoBuild can be separated as: initialize, build, and start, with corresponding function. From 5f32e7353d0f0cbe148dccf688ad4cc8bcf04181 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 04:24:11 +0900 Subject: [PATCH 43/74] change test file name; modify test --- autogen/agentchat/contrib/agent_builder.py | 25 +++++++++--- .../contrib/example_test_config.json} | 0 test/agentchat/contrib/test_autobuder.py | 39 ++++++++++++++----- 3 files changed, 49 insertions(+), 15 deletions(-) rename test/{test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json => agentchat/contrib/example_test_config.json} (100%) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index dd92deb79fc..2db9b9a30f3 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -5,8 +5,7 @@ import os import json import hashlib -from typing import Optional, List, Dict, Tuple -from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent +from typing import Optional, List, Dict, Tuple, Union class AgentBuilder: @@ -198,6 +197,8 @@ def create_agent( {"config_list": config_list, "model": model_name_or_hf_repo, "max_tokens": self.max_tokens} ) if use_gpts: + from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent + agent = GPTAssistantAgent( name=agent_name, llm_config={**current_config, "assistant_id": None}, @@ -248,6 +249,8 @@ def build( coding: Optional[bool] = None, cached_configs: Optional[Dict] = None, use_gpts: Optional[bool] = False, + user_proxy_work_dir: Optional[str] = None, + docker: Optional[Union[list, bool, str]] = False, **kwargs, ): """ @@ -259,6 +262,8 @@ def build( coding: use to identify if the user proxy (a code interpreter) should be added. cached_configs: previously saved agent configs. use_gpts: if true, build function will use GPTs api to build agents. + user_proxy_work_dir: The working directory for the code execution. + docker (Optional, list, str or bool): The docker image to use for code execution. """ use_api = False @@ -351,7 +356,12 @@ def build( name="User_console_and_Python_code_interpreter", is_termination_msg=lambda x: "TERMINATE" in x.get("content"), system_message="User console with a python code interpreter interface.", - code_execution_config={"last_n_messages": 2, "work_dir": "groupchat"}, + code_execution_config={ + "last_n_messages": 2, + "work_dir": user_proxy_work_dir, + "use_docker": docker, + "timeout": 60, + }, human_input_mode="NEVER", ) else: @@ -388,18 +398,21 @@ def save(self, filepath: Optional[str] = None) -> str: return filepath - def load(self, filepath: str, use_gpts: Optional[bool] = False): + def load( + self, + filepath: str, + **kwargs, + ): """ Load building configs and call the build function to complete building without calling online LLMs' api. Args: filepath: filepath for the save config. - use_gpts: if true, build function will use GPTs api to build agents. """ if os.path.isfile(filepath): print(f"Loding config from {filepath}") cached_configs = json.load(open(filepath)) - self.build(cached_configs=cached_configs, use_gpts=use_gpts) + self.build(cached_configs=cached_configs, **kwargs) else: raise FileNotFoundError(f"Config file {filepath} does not exist.") diff --git a/test/test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json b/test/agentchat/contrib/example_test_config.json similarity index 100% rename from test/test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json rename to test/agentchat/contrib/example_test_config.json diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index e34104cfa30..255e82d4c31 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -4,7 +4,10 @@ from autogen.agentchat.contrib.agent_builder import AgentBuilder -oai_config_path = "OAI_CONFIG_LIST" +KEY_LOC = "notebook" +OAI_CONFIG_LIST = "OAI_CONFIG_LIST" +here = os.path.abspath(os.path.dirname(__file__)) +oai_config_path = os.path.join(KEY_LOC, OAI_CONFIG_LIST) def test_build(): @@ -15,7 +18,12 @@ def test_build(): "and find its potential applications in software." ) - builder.build(building_task=building_task, default_llm_config={"temperature": 0}) + builder.build( + building_task=building_task, + default_llm_config={"temperature": 0}, + user_proxy_work_dir=f"{here}/test_agent_scripts", + docker="python:3", + ) # check number of agents assert len(builder.agent_procs_assign.keys()) <= builder.max_agents @@ -33,8 +41,13 @@ def test_save(): "and find its potential applications in software." ) - builder.build(building_task=building_task, default_llm_config={"temperature": 0}) - saved_files = builder.save("../../test_files/save_config_test.json") + builder.build( + building_task=building_task, + default_llm_config={"temperature": 0}, + user_proxy_work_dir=f"{here}/test_agent_scripts", + docker="python:3", + ) + saved_files = builder.save(f"{here}/example_save_config.json") # check config file path assert os.path.isfile(saved_files) @@ -52,13 +65,17 @@ def test_save(): def test_load(): builder = AgentBuilder(config_path=oai_config_path) - config_save_path = "../../test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json" + config_save_path = f"{here}/example_test_config.json" configs = json.load(open(config_save_path)) agent_configs = { e["name"]: {"model": e["model"], "system_message": e["system_message"]} for e in configs["agent_configs"] } - builder.load(config_save_path) + builder.load( + config_save_path, + user_proxy_work_dir=f"{here}/test_agent_scripts", + docker="python:3", + ) # check config loading assert builder.coding == configs["coding"] @@ -72,8 +89,12 @@ def test_load(): def test_clear_agent(): builder = AgentBuilder(config_path=oai_config_path) - config_save_path = "../../test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json" - builder.load(config_save_path) + config_save_path = f"{here}/example_test_config.json" + builder.load( + config_save_path, + user_proxy_work_dir=f"{here}/test_agent_scripts", + docker="python:3", + ) builder.clear_all_agents() # check if the agent cleared @@ -82,7 +103,7 @@ def test_clear_agent(): def test_start(): builder = AgentBuilder(config_path=oai_config_path) - config_save_path = "../../test_files/save_config_eb1be857faa608aeb4c5af11fe4ab245.json" + config_save_path = f"{here}/example_test_config.json" builder.load(config_save_path) test_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." From 2e8e912c3fa134244fe60c372a2a5a0e3ab1c818 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 04:37:15 +0900 Subject: [PATCH 44/74] modify test --- test/agentchat/contrib/test_autobuder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index 255e82d4c31..018169587c5 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -11,7 +11,7 @@ def test_build(): - builder = AgentBuilder(config_path=oai_config_path) + builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( "Find a paper on arxiv by programming, and analysis its application in some domain. " "For example, find a latest paper about gpt-4 on arxiv " @@ -34,7 +34,7 @@ def test_build(): def test_save(): - builder = AgentBuilder(config_path=oai_config_path) + builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( "Find a paper on arxiv by programming, and analysis its application in some domain. " "For example, find a latest paper about gpt-4 on arxiv " @@ -63,7 +63,7 @@ def test_save(): def test_load(): - builder = AgentBuilder(config_path=oai_config_path) + builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") config_save_path = f"{here}/example_test_config.json" configs = json.load(open(config_save_path)) @@ -87,7 +87,7 @@ def test_load(): def test_clear_agent(): - builder = AgentBuilder(config_path=oai_config_path) + builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") config_save_path = f"{here}/example_test_config.json" builder.load( @@ -102,7 +102,7 @@ def test_clear_agent(): def test_start(): - builder = AgentBuilder(config_path=oai_config_path) + builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") config_save_path = f"{here}/example_test_config.json" builder.load(config_save_path) test_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." From 379621586ac05bebee6fa8eb8cd9a250ff539e8c Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 04:53:01 +0900 Subject: [PATCH 45/74] modify test --- test/agentchat/contrib/test_autobuder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index 018169587c5..c2510f041da 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -1,11 +1,12 @@ import pytest import os import json +import sys from autogen.agentchat.contrib.agent_builder import AgentBuilder +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402 -KEY_LOC = "notebook" -OAI_CONFIG_LIST = "OAI_CONFIG_LIST" here = os.path.abspath(os.path.dirname(__file__)) oai_config_path = os.path.join(KEY_LOC, OAI_CONFIG_LIST) From 84bc57d8fdf6928f3c2b1f3782941b395c3c742d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 05:07:36 +0900 Subject: [PATCH 46/74] add try/catch for dependency --- test/agentchat/contrib/test_autobuder.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index c2510f041da..e046bc54657 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -10,7 +10,18 @@ here = os.path.abspath(os.path.dirname(__file__)) oai_config_path = os.path.join(KEY_LOC, OAI_CONFIG_LIST) +try: + import openai + OPENAI_INSTALLED = True +except ImportError: + OPENAI_INSTALLED = False + + +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + reason="do not run on MacOS or windows or dependency is not installed", +) def test_build(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( @@ -34,6 +45,10 @@ def test_build(): assert "TERMINATE" in agent.system_message +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + reason="do not run on MacOS or windows or dependency is not installed", +) def test_save(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( @@ -63,6 +78,10 @@ def test_save(): assert saved_configs.get("default_llm_config", None) is not None +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + reason="do not run on MacOS or windows or dependency is not installed", +) def test_load(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -87,6 +106,10 @@ def test_load(): assert agent_configs[agent_name]["system_message"] == agent[0].system_message +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + reason="do not run on MacOS or windows or dependency is not installed", +) def test_clear_agent(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -102,6 +125,10 @@ def test_clear_agent(): assert len(builder.agent_procs_assign) == 0 +@pytest.mark.skipif( + sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + reason="do not run on MacOS or windows or dependency is not installed", +) def test_start(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") config_save_path = f"{here}/example_test_config.json" From 53b6ce7a6757ba22746f7fe48297e2d5ed733798 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 05:15:49 +0900 Subject: [PATCH 47/74] add version requirement for openai --- test/agentchat/contrib/test_autobuder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index e046bc54657..af6a15d4e34 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -11,6 +11,9 @@ oai_config_path = os.path.join(KEY_LOC, OAI_CONFIG_LIST) try: + import pkg_resources + + pkg_resources.require("openai>=1") import openai OPENAI_INSTALLED = True From f79a48c8f6d1a816812db4af3e8a947ba792417b Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 05:20:06 +0900 Subject: [PATCH 48/74] add exception for DistributionNotFound error --- test/agentchat/contrib/test_autobuder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index af6a15d4e34..bfdf15fd20f 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -19,6 +19,8 @@ OPENAI_INSTALLED = True except ImportError: OPENAI_INSTALLED = False +except pkg_resources.DistributionNotFound: + OPENAI_INSTALLED = False @pytest.mark.skipif( From 891fbce4edf26c16a22c4c706aabbcd5ae34a337 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 05:54:59 +0900 Subject: [PATCH 49/74] add requirement --- test/agentchat/contrib/test_autobuder.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index bfdf15fd20f..aa986a01342 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -2,6 +2,7 @@ import os import json import sys +from packaging.requirements import Requirement from autogen.agentchat.contrib.agent_builder import AgentBuilder sys.path.append(os.path.join(os.path.dirname(__file__), "..")) @@ -10,17 +11,18 @@ here = os.path.abspath(os.path.dirname(__file__)) oai_config_path = os.path.join(KEY_LOC, OAI_CONFIG_LIST) +# openai>=1 required try: - import pkg_resources - - pkg_resources.require("openai>=1") - import openai + from openai import OpenAI, APIError + from openai.types.chat import ChatCompletion + from openai.types.chat.chat_completion import ChatCompletionMessage, Choice + from openai.types.completion import Completion + from openai.types.completion_usage import CompletionUsage + import diskcache OPENAI_INSTALLED = True except ImportError: OPENAI_INSTALLED = False -except pkg_resources.DistributionNotFound: - OPENAI_INSTALLED = False @pytest.mark.skipif( From 86d1e376699e4bfea6f021ece63623c9ba91e9af Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 05:56:52 +0900 Subject: [PATCH 50/74] change assertion --- test/agentchat/contrib/test_autobuder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_autobuder.py index aa986a01342..81a3f890659 100644 --- a/test/agentchat/contrib/test_autobuder.py +++ b/test/agentchat/contrib/test_autobuder.py @@ -26,7 +26,7 @@ @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, reason="do not run on MacOS or windows or dependency is not installed", ) def test_build(): @@ -53,7 +53,7 @@ def test_build(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, reason="do not run on MacOS or windows or dependency is not installed", ) def test_save(): @@ -86,7 +86,7 @@ def test_save(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, reason="do not run on MacOS or windows or dependency is not installed", ) def test_load(): @@ -114,7 +114,7 @@ def test_load(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, reason="do not run on MacOS or windows or dependency is not installed", ) def test_clear_agent(): @@ -133,7 +133,7 @@ def test_clear_agent(): @pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or OPENAI_INSTALLED, + sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, reason="do not run on MacOS or windows or dependency is not installed", ) def test_start(): From 99a03bd59d0f6c2bee37c16bde7dc37bf3cc54f2 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 08:20:09 +0900 Subject: [PATCH 51/74] rename test; modify index.mdx --- .../contrib/{test_autobuder.py => test_agent_builder.py} | 0 website/blog/2023-11-26-Agent-AutoBuild/index.mdx | 2 -- 2 files changed, 2 deletions(-) rename test/agentchat/contrib/{test_autobuder.py => test_agent_builder.py} (100%) diff --git a/test/agentchat/contrib/test_autobuder.py b/test/agentchat/contrib/test_agent_builder.py similarity index 100% rename from test/agentchat/contrib/test_autobuder.py rename to test/agentchat/contrib/test_agent_builder.py diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index 02ff6775552..3fa29990285 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -6,8 +6,6 @@ authors: tags: [LLM, research] --- -![system](img/system.png) - **TL;DR:** Introducing **AutoBuild**, building multi-agent system automatically, fast, and easily for complex tasks with minimal user prompt required, powered by a new designed class **AgentBuilder**. AgentBuilder also support open-source LLMs by From 3d713251b08d9bd5e0bdc305fc06099245473103 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Wed, 29 Nov 2023 08:22:39 +0900 Subject: [PATCH 52/74] change config file name --- ...config.json => example_test_agent_builder_config.json} | 0 test/agentchat/contrib/test_agent_builder.py | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename test/agentchat/contrib/{example_test_config.json => example_test_agent_builder_config.json} (100%) diff --git a/test/agentchat/contrib/example_test_config.json b/test/agentchat/contrib/example_test_agent_builder_config.json similarity index 100% rename from test/agentchat/contrib/example_test_config.json rename to test/agentchat/contrib/example_test_agent_builder_config.json diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index 81a3f890659..68d0d201e8a 100644 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -70,7 +70,7 @@ def test_save(): user_proxy_work_dir=f"{here}/test_agent_scripts", docker="python:3", ) - saved_files = builder.save(f"{here}/example_save_config.json") + saved_files = builder.save(f"{here}/example_save_agent_builder_config.json") # check config file path assert os.path.isfile(saved_files) @@ -92,7 +92,7 @@ def test_save(): def test_load(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") - config_save_path = f"{here}/example_test_config.json" + config_save_path = f"{here}/example_test_agent_builder_config.json" configs = json.load(open(config_save_path)) agent_configs = { e["name"]: {"model": e["model"], "system_message": e["system_message"]} for e in configs["agent_configs"] @@ -120,7 +120,7 @@ def test_load(): def test_clear_agent(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") - config_save_path = f"{here}/example_test_config.json" + config_save_path = f"{here}/example_test_agent_builder_config.json" builder.load( config_save_path, user_proxy_work_dir=f"{here}/test_agent_scripts", @@ -138,7 +138,7 @@ def test_clear_agent(): ) def test_start(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") - config_save_path = f"{here}/example_test_config.json" + config_save_path = f"{here}/example_test_agent_builder_config.json" builder.load(config_save_path) test_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." From be884423eb0d191886417b185ec97697313167c9 Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Tue, 28 Nov 2023 17:19:27 -0800 Subject: [PATCH 53/74] Update agent_chat.md --- website/docs/Use-Cases/agent_chat.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/docs/Use-Cases/agent_chat.md b/website/docs/Use-Cases/agent_chat.md index 56725d954e9..6a41bf1bdcb 100644 --- a/website/docs/Use-Cases/agent_chat.md +++ b/website/docs/Use-Cases/agent_chat.md @@ -127,6 +127,9 @@ The figure below shows six examples of applications built using AutoGen. - Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) - Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) +7. **Automatic Agent Building** + - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_autobuild.ipynb) + ## For Further Reading _Interested in the research that leads to this package? Please check the following papers._ From 5402f8feee99b77cbd02aa78cb49d13bd0217859 Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Tue, 28 Nov 2023 17:20:52 -0800 Subject: [PATCH 54/74] Update agent_chat.md --- website/docs/Use-Cases/agent_chat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/Use-Cases/agent_chat.md b/website/docs/Use-Cases/agent_chat.md index 6a41bf1bdcb..e9f2fbcca71 100644 --- a/website/docs/Use-Cases/agent_chat.md +++ b/website/docs/Use-Cases/agent_chat.md @@ -127,7 +127,7 @@ The figure below shows six examples of applications built using AutoGen. - Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) - Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) -7. **Automatic Agent Building** +8. **Automatic Agent Building** - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_autobuild.ipynb) ## For Further Reading From b429dd23e46336562b43da85a7a68603c26c96b0 Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Tue, 28 Nov 2023 17:21:43 -0800 Subject: [PATCH 55/74] Update AgentChat.md --- website/docs/Examples/AgentChat.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/docs/Examples/AgentChat.md b/website/docs/Examples/AgentChat.md index 5d3cd6b1d40..629f1159b26 100644 --- a/website/docs/Examples/AgentChat.md +++ b/website/docs/Examples/AgentChat.md @@ -52,3 +52,6 @@ Links to notebook examples: - Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) - Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) - Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) + +8. **Automatic Agent Building** + - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_autobuild.ipynb) From e8694366835bf7d2bd33c88ea70858e1f58dc63d Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 1 Dec 2023 02:37:32 +0900 Subject: [PATCH 56/74] complete blog; fix typos in notebook --- notebook/agentchat_autobuild.ipynb | 94 ++++++------- .../blog/2023-11-26-Agent-AutoBuild/index.mdx | 126 +++++++++++++++++- 2 files changed, 171 insertions(+), 49 deletions(-) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 0793bd15a44..5b5768dcb97 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -11,7 +11,7 @@ "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. AutoBuilder also support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html) and [Fastchat](https://github.com/lm-sys/FastChat). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html)." + "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. AgentBuilder also support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html) and [Fastchat](https://github.com/lm-sys/FastChat). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html)." ] }, { @@ -55,11 +55,11 @@ "execution_count": 1, "id": "2505f029423b21ab", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:18.987795800Z", "start_time": "2023-11-28T09:23:18.977219Z" - } + }, + "collapsed": false }, "outputs": [], "source": [ @@ -85,11 +85,11 @@ "execution_count": 2, "id": "bfa67c771a0fed37", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:19.902186400Z", "start_time": "2023-11-28T09:23:18.979228600Z" - } + }, + "collapsed": false }, "outputs": [], "source": [ @@ -115,11 +115,11 @@ "execution_count": 3, "id": "68315f6ec912c58a", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:21.419250500Z", "start_time": "2023-11-28T09:23:21.413220200Z" - } + }, + "collapsed": false }, "outputs": [], "source": [ @@ -138,7 +138,7 @@ "```python\n", "builder.build(building_task, default_llm_config, coding=True)\n", "```\n", - "If `coding` is not specified, AutoBuilder will determine on its own whether the user proxy should be added or not according to the task." + "If `coding` is not specified, AgentBuilder will determine on its own whether the user proxy should be added or not according to the task." ] }, { @@ -146,11 +146,11 @@ "execution_count": 4, "id": "ab490fdbe46c0473", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:23.564327400Z", "start_time": "2023-11-28T09:23:23.488457300Z" - } + }, + "collapsed": false }, "outputs": [ { @@ -189,23 +189,23 @@ "execution_count": 5, "id": "7d52e3d9a1bf91cb", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:27.331284800Z", "start_time": "2023-11-28T09:23:25.264410200Z" - } + }, + "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", + "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", "\n", "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers that match the query. We'll write a Python script that uses the `requests` library to make a GET request to the arXiv API with the appropriate query parameters. The script will parse the response to find the most recent paper related to GPT-4 and then print out the title, authors, summary, and potential applications in software.\n", "\n", @@ -294,8 +294,8 @@ "Please run the script, and I will assist you with the analysis of the potential applications in software based on the information retrieved.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001b[0m\n" ] }, { @@ -309,8 +309,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001b[0m\n" ] }, { @@ -324,7 +324,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -362,7 +362,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mDomain_expert\u001B[0m (to chat_manager):\n", + "\u001b[33mDomain_expert\u001b[0m (to chat_manager):\n", "\n", "Based on the summary of the paper titled \"MEDITRON-70B: Scaling Medical Pretraining for Large Language Models,\" we can infer several potential applications in software, particularly within the medical domain:\n", "\n", @@ -414,11 +414,11 @@ "execution_count": 6, "id": "7fb0bfff01dd1330", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:28.687772900Z", "start_time": "2023-11-28T09:23:28.681728600Z" - } + }, + "collapsed": false }, "outputs": [ { @@ -478,11 +478,11 @@ "execution_count": 7, "id": "e4b88a5d482ceba4", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:29.950261800Z", "start_time": "2023-11-28T09:23:29.944238100Z" - } + }, + "collapsed": false }, "outputs": [ { @@ -504,7 +504,7 @@ "collapsed": false }, "source": [ - "After that, you can load the saved config and skip the building process. AutoBuilder will create agents with those information without prompting the builder manager." + "After that, you can load the saved config and skip the building process. AgentBuilder will create agents with those information without prompting the builder manager." ] }, { @@ -512,11 +512,11 @@ "execution_count": 8, "id": "34addd498e5ab174", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:23:39.363905700Z", "start_time": "2023-11-28T09:23:31.493375Z" - } + }, + "collapsed": false }, "outputs": [ { @@ -528,11 +528,11 @@ "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", "Adding user console proxy...\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "Find a latest paper about Llava on arxiv and find its potential applications in multi-modal model.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", + "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", "\n", "To find the latest paper about \"Llava\" on arXiv and evaluate its potential applications in multi-modal models, we will break down the task into the following steps:\n", "\n", @@ -586,8 +586,8 @@ "After running the script, please provide the title and abstract of the paper so that we can discuss its potential applications.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001b[0m\n" ] }, { @@ -601,8 +601,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001b[0m\n" ] }, { @@ -616,7 +616,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -655,7 +655,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mDomain_expert\u001B[0m (to chat_manager):\n", + "\u001b[33mDomain_expert\u001b[0m (to chat_manager):\n", "\n", "Based on the information provided, the paper titled \"PG-Video-LLaVA: Pixel Grounding Large Video-Language Models\" presents a Large Multimodal Model (LMM) that extends image-based LMMs to videos by incorporating pixel-level grounding capabilities and integrating audio cues through text transcription to enhance video-context understanding.\n", "\n", @@ -712,11 +712,11 @@ "execution_count": 9, "id": "4051c25b2cd1918c", "metadata": { - "collapsed": false, "ExecuteTime": { "end_time": "2023-11-28T09:26:01.069144900Z", "start_time": "2023-11-28T09:23:39.355869200Z" - } + }, + "collapsed": false }, "outputs": [ { @@ -745,11 +745,11 @@ "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", "Adding user console proxy...\n", - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "Find a latest paper about XAI on arxiv and find its potential applications in medical.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", + "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", "\n", "To accomplish the task of finding the latest paper about Explainable Artificial Intelligence (XAI) on arXiv and evaluating its potential applications in the medical field, we will break down the task into several steps:\n", "\n", @@ -812,8 +812,8 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" ] }, { @@ -827,7 +827,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "exitcode: 1 (execution failed)\n", "Code output: \n", @@ -838,7 +838,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mSoftware_engineer\u001B[0m (to chat_manager):\n", + "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", "\n", "It appears there was an error due to the use of a backslash within an f-string expression. Let's correct the script to properly print the summary without causing a syntax error. We can use double quotes for the string inside the `replace` method to avoid this issue.\n", "\n", @@ -874,8 +874,8 @@ "Please run the corrected script to get the details of the latest paper regarding Explainable Artificial Intelligence (XAI) on arXiv. Once you obtain the summary, we can discuss how to conceptualize its real-world applications in the medical field.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n" + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" ] }, { @@ -889,7 +889,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", @@ -901,7 +901,7 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[33mDomain_expert\u001B[0m (to chat_manager):\n", + "\u001b[33mDomain_expert\u001b[0m (to chat_manager):\n", "\n", "Based on the summary of the paper titled \"Forms of Understanding of XAI-Explanations,\" we can glean that this paper is conceptual in nature, aiming to define and explore the concept of 'understanding' in the context of Explainable Artificial Intelligence (XAI). The interdisciplinary perspective adopted by the authors examines the dynamics of understanding during the process of providing explanations, particularly focusing on two types of understanding: enabledness (knowing how) and comprehension (knowing that), ranging from shallow to deep levels.\n", "\n", diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index 3fa29990285..f121d67ddb1 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -1,3 +1,4 @@ + --- title: Agent AutoBuild - Automatically Building Multi-agent Systems authors: @@ -24,5 +25,126 @@ AgentBuilder support open-source models on Hugging Face powered by [vLLM](https: and [FastChat](https://github.com/lm-sys/FastChat). Once the user choose to use open-source LLM, AgentBuilder will set up an endpoint server automatically without any user participant. -### Pipeline -The pipeline of AutoBuild can be separated as: initialize, build, and start, with corresponding function. +## Installation +- AutoGen: +```bash +pip install pyautogen==0.2.0b5 +``` +- (Optional: if you want to use open-source LLMs) vLLM and FastChat +```bash +pip install vllm fastchat +``` + +## Basic Example +In this section, we provide a step-by-step example of how to use AgentBuilder to build a multi-agent system for a specific task. + +### Step 1: prepare configurations +First, we need to prepare the Agent configurations. +Specifically, a config path containing model name and api key, and a default config for each agent, are required. +```python +config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # modify path +default_llm_config = { + 'temperature': 0 +} +``` + +### Step 2: create a AgentBuilder instance +Then, we create a AgentBuilder instance with the config path and default config. +You can also specific the builder model and agent model, which are the LLMs used for building and agent respectively. +```python +from autogen.agentchat.contrib.agent_builder import AgentBuilder + +builder = AgentBuilder(config_path=config_path, builder_model='gpt-4-1106-preview', agent_model='gpt-4-1106-preview') +``` + +### Step 3: specify the building task +Specify a building task with a general description. Building task will help build manager (a LLM) decide what agents should be build. +Note that your building task should have a general description of the task. Adding some specific examples is better. +```python +building_task = "Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software." +``` + +### Step 4: build group chat agents +Use `build()` to let build manager (with a `builder_model` as backbone) complete the group chat agents generation. +If you think coding is necessary in your task, you can use `coding=True` to add a user proxy (a local code interpreter) into the agent list as: +```python +builder.build(building_task, default_llm_config, coding=True) +``` +If `coding` is not specified, AgentBuilder will determine on its own whether the user proxy should be added or not according to the task. + +### Step 5: execute the task +Let agents generated in `build()` to complete the task collaboratively in a group chat. +```python +execution_task="Find a latest paper about gpt-4 on arxiv and find its potential applications in software." +builder.start(task=execution_task) +``` + +### Step 6 (Optional): clear all agents and prepare for the next task +You can clear all agents generated in this task by the following code if your task is completed or the next task is largely different from the current task. +```python +builder.clear_all_agents(recycle_endpoint=True) +``` +If the agent's backbone is an open-source LLM, this process will also shutdown the endpoint server. More details in the next section. +If necessary, you can use `recycle_endpoint=False` to retain the previous open-source LLM's endpoint server. + +## Save and Load +You can save all necessary information of the built group chat agents by +```python +saved_path = builder.save() +``` +Configs will be saved in the JSON format with following content: +```json +// FILENAME: save_config_TASK_MD5.json +{ + "building_task": "Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.", + "agent_configs": [ + { + "name": "...", + "model": "...", + "system_message": "..." + }, + ... + ], + "manager_system_message": "...", + "coding": true, + "default_llm_config": { + "temperature": 0 + } +} +``` +You can provide a specific filename, otherwise, AgentBuilder will save config to the current path with a generated filename `save_config_TASK_MD5.json`. + +You can load the saved config and skip the building process. AgentBuilder will create agents with those information without prompting the build manager. +```python +new_builder = AgentBuilder(config_path=config_path).load(saved_path) +new_builder.start() +``` + +## Use Open-source LLM +AutoBuild support open-source LLM by [vLLM](https://docs.vllm.ai/en/latest/index.html) and [FastChat](https://github.com/lm-sys/FastChat). +Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html). +After satisfying the requirements, you can add an open-source LLM's huggingface repository to the config file, +```json, +// Add the LLM's huggingface repo to your config file and use EMPTY as the api_key. +[ + ... + { + "model": "meta-llama/Llama-2-13b-chat-hf", + "api_key": "EMPTY" + } +] +``` +and specify it when initializing AgentBuilder. , +AgentBuilder will automatically set up an endpoint server for open-source LLM. Make sure you have sufficient GPUs resources. + +## Use GPTs +[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task. +AutoBuild also support GPTs api by adding `use_gpts=True` to the `build()` function. +```python +# Transfer to GPTs API. +new_builder.build(building_task, default_llm_config, use_gpts=True) +``` + +## Summary +We propose AutoBuild with a new class `AgentBuilder`. AutoBuild can help user solve their complex task with an automatically built multi-agent system. AutoBuild support open-source LLMs and GPTs api, giving users more flexibility to choose their favorite models. +More related features coming soon. From 3661eb4de5cb6ca102da78155416909088081a89 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Fri, 1 Dec 2023 05:13:00 +0900 Subject: [PATCH 57/74] add autobuild banner --- .../img/agent_autobuild.png | Bin 0 -> 269313 bytes .../blog/2023-11-26-Agent-AutoBuild/index.mdx | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 website/blog/2023-11-26-Agent-AutoBuild/img/agent_autobuild.png diff --git a/website/blog/2023-11-26-Agent-AutoBuild/img/agent_autobuild.png b/website/blog/2023-11-26-Agent-AutoBuild/img/agent_autobuild.png new file mode 100644 index 0000000000000000000000000000000000000000..a6645b109d2cbefc02fa8e821b10157cc0f3959e GIT binary patch literal 269313 zcmeFYcT|)2-#<+2ZMmy$`MN+wp=uQ(0xqD;v?`)Oh(OsR2qFRkVt^1r60NnC87%>1 zMuC9Hl05>63!=;tAVEe_K@uZ`7!nd7>vzS0c;CNsp1+^xJUORT@Vefg_k6wACy8er z?Y>{PeVv+`+V`hVop4rDTj!*v_D$D+e*^r+$LaBZ1Alx4ceXpGR@JwI3;gh3!M{5E zs-{+hTQ57e7Wnzw%cnfyYHEicss8&)|Gd|jnwpY$`oyol#re%ns5eKudz#Ii-Mkx; zQFZr}I@ckdzxR)Sj=RQhxV7KE__UhS?H_1iKi*3}cIx+}5xK|(ycYk%SKsdZ#>obeJL}&s0;jIagO{-ALb7HKfv+-|L6Zh8h9#x?c^LZxx*io z!DQ5Zjk;H_#kf61O~mqdo9(!XfrO1_j4BM?6%=|NM#V5)bW9`nq7@uVd}{%s}@Lx_=A>pPb z>p)7o5Ok|E3TN0bkl{eVW&xZlb3YNttPowr6g)D`b$zb64e|;?Xb~qKE9mIiKbrsk z*w+us>rDZkOY!e{u2yLtW&H8<15S);!vZ~>w{@wZ%r(bfP9-b$^R0SH+1pib<>oba z;u$3dN&3dh9${)2VX@1fFY_gnSDs%!2|VS}Xe3PRDQ|JbCeZ+xWR| zhr5p>zWVU0|LM^6&zGNFM>oE}94M?X{C8&4TmAPhmIOIPVgGt2#oU%S)IScz(@6lU zAK$u4I2(VElJcl=B+?dtNI`4UO#QvUB1xa);*}fX**Gfg_=%}o?&8I-y0jdl5BQRJ z^Cz7=ZtM6FJ^9LJEP%4bscO7EuH-)L4rvne$?)qR7^Wpng{^;RI{g(k zWe?4@elrZm2u#b3xU#dPH|Q|oxoNg(Q-WW&o78WlF{U?z-e?eYNx$9HP~-wP+&yE8 z1jn}P?GY?If87jdAkfRLjJ>_HrsAaPum|W-PW#5{{$j4wNXtPSpUey8+;+wC5| z{i#g`=?j&^$`Kz$ZR(Y6aP|BB|Xm_sj8Sqs}h~+b6rb=Z?Ey1^f&l^wc*y0ny>jxB)!e}3NH-X|)sGy4)J$7TC!%%ZpH})iZoIH2DAYsm=<9xEF*^tOJ zH^?_Vr~Q_9$N2orQ3swPUk^>KjUd*<65H({)HB@==yz58pf%`o86gm=@sZ@V(kyiU zCB0D?8>7E)$gupkx>(^X)!EYbWwG2(g^LYv5P%W343K!fmUU!FS|o~h*UbOqF|%RB zn}R`_o5ck$`mUt^K+wi}(!ai$W1|U*xNmqN+YI@t(<-#nye4htXs-o5b|A+0#wfB$ z!Y||)+m}^W9kBRw)cn1{vP=KElELqA&TPx8jd-&qVWH1GYzK9sVJ}gnjQF zPVerp*8L6A2jK-_P3wr5*4j7d(%9-VDz_;!EcBtGYVgP;>oEw=vHQW$EDp)>4K4rD zJcs!fl~dL(W7I;%CnpwHjvh;N{?#K~yIVgmyQJ}#OU$7jRKet*qcOg25JodDM^IIM zF^ul#P2Jv4Dk%SLd$4;e{bBTMeMP`dh&(!|K;mEZiHjC$)lB~5HF7;p0f&TxD}u=?jaLX**%HExF_Z<;0$_3h6Wc(o~q`XM}vDra6oX{GP| zaN{&siLj%i{ zv8IQFJ(ZP15$>jFO4cXi5KDNYCfUXO211kkbSUH7kGuf7zqx>3Hy%3XbJFR>Gpn_h zXf{5#>772UsrlR%#$I=lqe&>Wp!9-HIho$TXyG?Z{1;JxAo8Vk(R1}fb1y?6JVemI z%x@%0Smk{C%f>Wm(4A7k=*(}SfrGUbs1s#8AmL$*0hDl^OU!9x2*_n}iC%xLc-uz~ zX0?gx?*?h^-0=`k`f;W6*x=zvig2^jiDi+yx7CKgY`b0nQbB2k4TlVbn}ah(vomv$ zLw+}r5?br9AAu~);j{v9!K*qDpC8KcEEh8mS0j_^*+`t@P+z&2>=qj6vJviio;M0X zM+Xf`P%GdbI%<>;Hf%s40}3P;&##J=uKmf++3xA4h+;^Y zXZfhWE>!GR6-yWv)pbdzeh+->$ZXJ}r`tKtk6cI)m9D;o#!=*f_)52SFoY8c$wWu0 zyxW1r2LRmJGRs-pmpc~Ai$AbTITFfOzr6~t3>rGrOx7AUgxhJldS7~|W&0~SNjsEX zbFt=x0}Nd$9A$`I>--uRh=H&gBVrAL*r;}YsI&e+ST7_TFojK+#HVspI%36cK^PwA zjSkOJXiK@7ca(+7xj&r8d|!&>ek;gL4rh?TTE0qGIsdzl_V7Qy)AlIQv?ORiU1`$s zeM84NUjTFM3Feu7GhKyi%?}kglZc{r8_Wsup9@HL2gqxWp&pmd|{husk_2p*<={Hy!Ci?-+;rq{sS?Kyb zY@|NS`@8y!=K{-ms}7Vu_kTI#;oaQy9WmCPi$c?DnywH{NzQ|e|0L_%PZeZ(HH2w; zp1kH)Hu2sS03$dYDD)|D?X8ZU+ZTKr%ZKn{zOestqpwAiL9!w;Cvi0F;VSGX68($z zVuPX|px^0cH{A`bUdPPz9X~t9Jd7VT405%_CujW2-G-6V4x|e0yBlq-vg3AJb#J zHA~OiLf8M!Hr!ukDDtSk92xMk<9idK$4P;we#1nD4IOO=36CX~G@rkC!=QoRf#e+N zmj}5>emTukRJo$^L&KktjTQzo$dVRX%h)aC+!QgkwU#Sw2P&FSjw)}XN2`33ZTVrF zzBo91*Q4DUufwECJw=U#p#2T349_r6?_V0`L{2C}(}t@Nz;VPw(C*z3oDdE`;X~BTE?88?8UodA+A zGjb;Xz6bo(57!Qvl_N7of$(C43`@N?OhZd>G~{4yDA6XESWRpBGGG9YnMRckVca#_ z*&kd+?cl`*+jFSw(cd53+iV7CLuDD0j=jf0h2tXYjM9F^eo({k2N9ZfcM;C$5``r} z!|WwVBVS>hKlUXo4T}B9i1(}<|3}r-)1HbQ936H&0#sw#{#R|kiKnDNU!@>x zlRtKMjfT~H`yq_t^WA)3H@_8!Nn%(~y9%ebr3N)RXgmNPG56au(UJ??@lVNSv_(~x z5E`fM0P6GnkTG-Cd{EaT|6$F(OUITq`~PU)|J?%~!Y%5!!(&MAa>ZcldWVnFy=251 z4{81qevFNb%pB#uH;rvjc?7n4E~CX6?8`+F4Du}vXU?LGeL8&1R%cr1SOWu|&Ch?I zrdJ<&>=I;IbJwivxwGL;9Tk&*=FAK={d-{|lyTlkg=8*1&ZJV)%FD!9S?=FRaf}K% zR3iV@_*Q)U1B_I^MY~!cfNHwa?=8syEy4V6i6w8b!Iz{I+mEjCa6@Cmo&C-(XI+7) z=-z+$zTv^_;m9^Wi)wsMgJ)@u`3X%|mA5d^W9!{RCG35~#>M*pPH3W8_|ZH7R2ktq zD)g9@p5aL!@cr~Z2yG~>idl6w>JpGuMIoqag~MO5D&HCWJf%oID^uoKWAJ9x{P&5J9&KYe%7l$wOXLX=x)aXpZMB{mA7_XUZTAk#En5{gf}} zo_D$o=D zSM0vTLyjB6yhyr}Z3!uL?-}CODOv!UW#OodQ1RLG1L&jnyn?@mW=XWAL7;$PJB9i3 z397)NEAHoe_o5y|E9xt}c&LLk1epjg4f2~Ad0$G@=^j(5KAaTmpebj$2z{=P#(ZiA zALnwul0E2k@p)feWj8%KS73f(d9ztX`Kdj9Do`We*w!!w`whT6QU22ts=}zT9YPd1 zbU&c_kr=y80w18kedR;+iXktCv^2I?b7GHh_F4|gn=1?PBqEjQH({qf0H4JFJr}3~ zblYD@F|CmA`q4gH^Jr_tA!Q&h&l6?*ki=uZ+k~#awYvdc(DchbpvJrx9pS!Q<>1Y; zTCQ^8nRuvp8PZhs6kR2otA?tSZPHRVUh{6W#E0is!13xmqBvDKieWc&%{V!uQbQ#{ zWvL;a8KX!?=Oe6LJc&xB`F1Kv>U}M+UZIltXKkpEf|*Z)x9cX*4bNJ8Tz33pw|NOb zjtsJ`x=_>POzi_qX!23EXDK;UGEu0)fW~Ika42-lFM~Ex668Bo5Zu48)00+1AK(EYQom&&b8mq9{Pi~3d*O?Vg`l`ye8q34@od*wB zZThH{@4UFLmg_5okEK?&|Ba?LT@?IE8rel|ym+nWCp70^|MIh~(#@egE;y|Sb$A~}hDM--^WuRNcRYVv=`MsB4<2sghJHB4p$E!b_vVoOFhy(O{Z;+ylCU6JA04avSHjmum8bq`R;&Z zhx^+f4)yIjKOu?vc)F7+M;F4&4QD-7@X%Ao$ZD8Nu9`V2Oo=icp1XaDCFy&>)LU98=LYI@ zw*?O~;hAVkUWV;lVvF-w?wt|K?i~1t_(R(x;R6uiDZ;8(zO^_5upqrMu)+e}J1C0w zpvNWQAPz|{$#;%A#%-=T&`K}gKXm6OyA3Qgead2--vea&oP_VdG? znxPDY${w|+qFx-PCv0KA>`$(ET8r;AF)%?z7sP;wL!AA|T~`JWWxY8Eu2h=I{n8B< z>b^}}@TN+`y+3rpRe>sN%xpKDveurNa2uSn5Nm&ofJ+rf9cFdmogf`zO>fuk0OoCd zQ~fP}J!33uQtJDG1XzWrcXb)3Wd!lBdDH>*(0kJy@bYoCY_dugPY`1@o(XdO3mWo* zzvrNFXHc-A`j-2)OimQDmdUCS3)A-0_SE3<%RVsyj>KnPm2c%X_9EE!nTpxr53XXO z+CBuqsxMX#_18bMK=|HVjOXS#i$nPzV@6H-o5Q)|=Ur`23`A^pJd+d$LEB9>{A`p4 z8}<-i_YAZ4i=Iu?Gqv9hT=4tJ+~D)0 z_U_WMTj>W1Oa3~XWbuhtHMPtLCeRt#WYhw~tHOUbt~~F*7t)bjcQ`UCFroH*S$ma! z;eW0Cn_J0mZ6kUOgp)uQw(T=1kL<;Is`FQYgcIk^Z+6LCw|^>ZKy1~Ke*0&Ql~?_P z3P);wd)CY-lNZ~s{Pt>Sa>DlM{lJc4h1%5y(3T@}pVBq(B>jR5rm_h(^(kelsZ~t) zX*ut|`+MlRZ{Qyz}F!+D-D6)6_2IE&lO*vG}?3SFhU=#ham@ zHW6xS-h0;VFIqgb@y4yXBm+I_nWTMN96$f-XA1wX{o7eRVx3NO0Q}l@8@kV{yPNu( zU;U~p7x>yK<p0-Py@4Rl)2p@mL$D;oMSrRPXyv}#A!(xK*gBlh@bKzYaG5X*B zHCSCVTCxC5oi|hOOBNR8r{~)3yl6Ix!#JMi?2Jjs6nq*Rs0{b-2z!Uq^Ei5Xjx>Mz zb-q^qG@gwvk_~%0M%Y+(N&4mm_`dn9B3Xw%)~1h;Caj5a1f8EBeLeD`Wx%msi&xMl zl&_KJ(PSNO`U6icwbx-PTKca&>Dv0K*j7_}oR1Ir{gG+ z>gmW_0{#}EkRB*KYN?*I1ANHnvaHUjb=^-{85hCJiHbtj8*t}_Df5uJHqJkwJAssYLPwWw4q%o4RuE2cE3h_QgtRMN zKkG&QvM(?(s$(%APY>6Lau#Z+cPa;tXyBGofb=$S~s-FCT> zRE1SIK7WmEB39Eju_&+JHc`;o4A#Nw_D&V`GVFvF``CzoPWM*txbtaesiyW2hb}Z< zyrk=JGmlr)o7*4jEL^ISBO$@kK_|y;)32QzxBn8LbBF-e<`lE*f-PH+gKzpRsWPg; zXQ94hZs1mcj)!lSiEkY9N4T#Ulifjt(SxaSO@qzULs?~ zxY;%d@P0(pKOk=2)RxZ?-`M-oF*E`#-jpLLkOORjV;~Ul!hn;_?)=1sL};m16ep#Q z@DSc}EnpQUpz`EyM$KXTI{1Mr|zX)C@dfvO+<5=NdF=m$oF!4uzQGhko{?oFo=g(zGy}MSur)UD#(E_*XI+vT>*`itW-Ll^`63 z=izDQY3U6?^-M52L6|YH4BlJybTR2OaFvT)bR#eec~ldDK*kEb^wBMMQXlzV*SX{zb@& zeRBv73>EV5NJEesN z2(Oq`8W+eOUh>)3f!Ad&4HD#M;Wq@tkCh1U0z2ki4cKNj*k&j}5bSt{8*Fnxo6zj2 z8%ne;;juINgcfx=h zLmxl>B`!2`4TZn35RPtsY0jtt<N9C-~n?VCHI4c0jXVvYNu3kbb5v)y#1T^J4BkE4z_QWH7EgjgEm8X7OH-njL z?@a_Fd>~*6X4|E-ok7w0SoR%uA#kN)5f(p7Z_TsEMgTJK*7i%+F=W!2}KlA-peaE8Q0TRnX=W1v78(edb*)X{WiIlP&*9 zotKBFoLjjsKohU}om+(jzG52|S52LX^4Wp?x6SDrUyp4JWxIoggw&VtIxcL5^+%WU zIo7@RiV`?&e0y#i^}8dScmxEDiiTt2?=y1YG*+sokgM}l1rHCIR(?ZCof}&hf>dMRntoR`|h`U$k z-<+EsFOv0;VwKWNy;6~#vVaCtxIHRrReeL9e#jzl=%xj!k-Oz>@nfQyIe zH0a7tU{G|)_AN94{GOOE()(82IwVSD+~ z{1WaoSLXE2>}sAgALRS`4Zc7QCEZA6$(&l=M7bs^$fnCjmT^(f;viGwT`A%hOKG9x zXeOcnGc3?t!P2Wd_$xCrPU|D5-YBg&C`2ocaqvn3q9i(TAVKkZ5QG-q1m)BBU-oea zN(W^_5kM}$2N=l*1yFVj@6eS7w`Hs_=TFL0<_Ft^H~ko~HSt&n%xWLeEWv`vFCZ_& zSk?yWn0Cp<9fu0GGOiu#deYo>s0yO(h9BehzFSA}<9^x}zwS$dS0P z6>ojSZhh{pl>QA?Hs<&Hyqmc3d$VEB5-XEdy(6DSb)1l^I*6%~pE@O_MNxk7N10m@ zAvWGwi!tvaSxd9F%u$~!Z5=BD`5*CpS7~XS_M~Y&8NJn9;_bZ56I2U~wirct1>`|- zuo>&^s`s-;ppg?xPX(GQ3FK-o{94md2Kk?{T)xknogm*g;bR0LWiK&|Y)r%|E`Fx%FjfKdUfxfg+EZx)sue^y)|w(7zQJr1USCfxYw3;a{;XLhNmSEo zZX~0#-z2vkA+EO@V5(hFm=U9uE{%NP_EusPbumzfxD*~3D{+VS>}B2RGiS!HNb#`A zNSw*m)I!<|r;^)Dh{-2v#?-D{Gb4)9ppl3TK`F|wBMRrCc%E9UNCt$Zf4c# zt?zOXX)SLoom-#h-n_lO9YpNb&7dh0VbkqI;Og1jR^^tH!oQaA?3UC-bnL69qx_wb z)+?uvSZZOPZex(%YM!OV@XJHk(; zSYn8hO?Dr+sZ5l|zB1#r#kZ|pk;Y$cG+)AAr-gzGGm_f@UD!sAscp24*Gf1J!HRGo zG(Y9awaLbxI+b?HdP~BZj+@6{c%MYV;cspI7(P*LTRu~2H(LIa{Xz?if@qEA zRveuO=rj`xs~vN~V^6otZS=cvK6ChGb#GAXKywkRR^*63UrlA6CVlfSUk}M zJWigYB6dceUeU1!F1a1UaiA>H@t9W&PrGbkSv0`FNs~{3S`7$$_J4zm)`dZ#?IyFRl}=hoR=m*rn%7JNhF!W04a@=;uODtnV~ ze=njO?yuw}%mq}37+ErNR(N>mm9?18#$?60Kq!l?ucjxlf76J}TeLB7HVxrutfd!W zA0Xpz?cMW%${0M+45vs4gewK~l@{Xa@Nx%Zu8p%X@(fq2=wPJ((hVmgaSq;30ex@g zD3zIYh0P;j2##wGzmfCECc#(8{DMNuYi*h>M$pj9FcvK1jW*a`TGO)@mO(qB*bUfb zNzbWsk+76F0r?b{x-yopQxY~)a%N!MTn!FS+FqFD zxa>)ob8Fw2^D<}9d6|vtvQ5e@*eaoc{^4GXuEvc2NZ!t4D#Du1G%EOA@1EaQA0L*XFwYR9# z`*VSsIg{l#+{SYT{KD97VD!sBv_h;9*BAf5-w~{$mUyS`>b47cA^7Tezdu}}7vARkFVx;1} zPjfRU@qN}kG)GJLCU=dr_T|!amf20*7fcvaE^(gQQ!I!uWQU@AuU`(W2zrV;q+1+4 z`lrH+4-^pm0V+SO=_u}=o|toyK=yA7`ix}enCT!(ev2r~WjP~@Pw@%G!{R+5#&S}x zyn%VBDl4^9(k=KFUn$^cJyWy|b;mL-nYAk_c-U+}ZZ=mEVf2K_!_^Qj&3KFeuvij3 z%qSLJgO-YT-y0OC2%a?uBpm9aElQ3DU=>~VxsKIhtEK8tfG zOI{TH)Lt1SZ0|mCZ`59bz;O41CKkU3-1*mD&9-xZVooXJqXmzNH7AJ8n zyfaVim0*`}{mlb$GoNh~fSJY(?#e<3-Mbi0U}q`xi8F1c#3m&i!m_X?4q`lvqJ3No zDUrZGG^H+6#HzB7_Om4LhuJ}J=$r+!bw$93ce=?H$Y;~QhA&9zv9D#w-no_5?GpbO zRxG-~pHq18s;ist90rDMg_J$u0@@ZkIrV4A)L6CoLnw@975b%+$Pl5dyM117Jzb14 zlK+i2TCf96OaaCAV)n#tDli8HhMF#Zv}`*S-Jv&$s!=7iXVy9sefxZcCw~s1o6NbO7WYvpTVPv z;(Z9!OqK!>eA@9 z5*GmPW3jH1kfH=@>E^CeGHbFzS`XGlQ*jN~0YiVjPb+leJ`i!9ltb!fdy>*Pg;h^1D%j`T4;Z7B<{Sfa#Y_ez=^ zp%0t7yG*>zuvI@&v9#gn%&(eNtf6@&)Vt z1()=aehvgcg|ai1|JNo2RaM~w2}I<>wRI4t2K!kj{snw4#dm@ORFgsfu?(I1WZ*Z? zFJEckVBj2xanfYHJiS{dn>Os1M+;%m);CCS6h5c z)|@2c(7prKcQpZCH@vJ*!9G*n#D3J6YvrS>+@f~%S_^G`u5M=1@Sk@K$d{Ymz3T(g zzqRtPQNj3KXhB}a*5DMgPP7PI8ck*J`t;a9Mf9WaN&X;Jg%hTKzh6dOz2x?1^la?~r z*7ZKzwI7}0e^Gqg=l+fFiKXB1MiLZgVs;B?=K_>O*=T9gTk~69bGZ4tGJMfe8~kW> zE;-?7KZoDf)cnKo=NAczx3?`z(^OgH(oeQ{71XXN`fH<_|45bW_k5`0?RuBxVXg0u zWGUP6bAl4QVtzd^)kwK=XtGhyCS!gRLAO_!N?Kb)jM;jk@YekEkD;l3dV{J}fAmPT z9NXGx0LF5vrx?F#CDR*!cDFf9?6x*0QrkFK3|hiapqkC!&9p=9xdYVMK1(*tD=XY_ zcF=%wcWFiyZ&y=`0fqoO`>Vbg^WB(h0c!_pSc@2)vFB#G0T@g)ko!TP(z6hPG!_C= zo`b;TkKe&ZFBTkP-9hw#>G$ew#&SkVDeKfxV1u4grMX8ZodKh&sU1A1 z`fAaiya)Y}1=t3FyAAlCbh zWsa1(BQX|NR6*mv2lRfMxOJ<_7Efa{I3~E}Wqv6@tR5gHcnJiY9E4U_g7TMHB*KXn zm~ePN7Pzy($_wX_MT-yj2GSExl4JLOwdiu2uI`lut>WkgPB^0Q?7LrqUOJsMB5$Az zPNzt%YksKF(ZI~iE#v9oxmQvLY}z~FxupLqrTM%g5plyD6BcQdpRu0NxXT?A3%S|0 zpGc9;&*>e+2vCPlS5R3v`d+#oy{sPNeS3t^Ke41RA*4tAtOA^ew?BN%>AMslGOnH@ zv4ynhYQ;GF2;(nUmTa8Y?Mg_eE%@D`MJdVN2N!SfPg@GgdvWq6JchTHG~?<;b?Bc{ zT(Ute1e>{Ujr0X3XQw`Vwde>kv$M||#349vlUerMI@WE^#F(Uoq?qIdgUXqbeE^|?^t2t3o`8TJs|9@LJB8?P z>4yuC#+WJuW zUOKQk*cYpiCt77D1;fC`=kaKrDcD_}kDrt0;1%335PA*;l5-Hnkpb-i69Mvoo%S=N zsBOINHBupxJ3nOz6fN5JtHYga4=ypMnh+X<1O~9TQM}Dk@5Jwq zp!P8;cIK3Nq!~{yjziZqH1dQ{tm>1lrk1Qy`K$&*F~Q!zK>I-$HUV&WWfW740HMVw z5c;k6;YgK4dSPB(e23EjD?10QuC%!VfmpOU5T#EW(k@nED_Jj>_+eB%<(73XO?SDY zPC{-{_7Sl%F9<>52qDNadvZM}9#oHh4+bqh^2FqzKJKDIn$vGJ_03cz(1~h0tWwj6 zZk4p2{e-cUpHFF@s{@)*{w2RE>#l}^Y??jPk$Z&Yz-3WD$@}Ps=tt-#^rrebRQ?E# zVY6F4p|7p0C5mgy%ioMbvoR$1iY~l0G{3ReE-0RzI0N{*!hgHRF5GJmLHCmpDs|3=L}_rt}C4^SxWjwu+} zArG?ydR_g1v0FYfO)eH94VPtMf2k;v1>wTTySgD2!4+W@krnU|xnzHhMzu7BZ9P>9 zfV20ns}&EMbk~D{lk^E>^OjB*=Xlvz=M8KrQPf{s|17>!&)Mb`k2K@sr37wyKb^5^ zZ(+rS0Wj}&b`{Kvvu13HT6z{Rq@|t(LN=}18qJ5dvzT&WP^qdDcjPJy_H{c}%yiq$ z>u; zMd_x3HD^JM&#HRb3e0@0E0Ke7_5fS-FW4fe*b**aN#^opLE=?PdsEAti%V-aki~CS z6Gb>!mg`=+gTyy@I;k@*w@^2MSEk!YD3r4;-W7sGA&v5^Qh80*blYl(o`h}jj zt%S4H(D(NHwsOQgF3FayjW7Ch$5r^R@<$4b26Z(}g! zEt9ID;UhOy`SmGNi>tM*h{)+wLILf2LM@_&xydD-0WKEd08c=`(gB{#=;m~lXc;4s z3@k+WK%lA8k*YfSRE*5R%;LJXnY9hL&6S(6yyJ%2FfokFc5+{FM|PEO&@73 zv5OYcVvK!pu6p8^sRg46l4WRU#8Kw&fRHg?2+2DM4NYr#G3eatw7I3y+$=5tf{1g2 z=qe<5g`ys>V27zRuVd-O@!)$%XbT=O5efbT6aF^U)OZB)1t!Ek)iY_@I!#qyH&IZ} z^ieiBTZkmi777=Td;&`pF<==g<^KW2(WOhI8NXH?gWhZMVc8;JfQUn?8TiADFL`Uk zg#cxgo++Qq1sqF|PS4erxj`feze04EZ~$)<=jHP1F2D-fGYS1#X>|Fw(=#<)vvbw# zH)G^tbE;}G{>W{0$^l;b_Ic)C{-xp8eI)CU&vZV^a6g?L99KyWz>B%dx*3ET-d zEwwePiu&M+KP*+uNG%u{k8hg>Y!S8^y8h5s()-3D#Y6arjkb|tUEk{xXlS64+oU`~ zq$~qa{j;lEh6l^zlk!{xU=xdpmi(=eCV=)rS7}e})k$1fRa81Fl#CqIPmoqFy#5ns z&m|j=%1s8bOT&1@q`?=A?cXTnmO#gs=)D*NU$|-y;$Oa+wKL<30?U5N%XDBbyNRBz zEx!PfpwB_{WOfk3VkkscnN+dAhFsHE!>pMj&eSA5!?eDbqTQY{1on>J&{fgoKgq^L z8esNu&YOYIvT_i5*8U3<)NTassrL8kjgYq15-%k^acRXf`WX0L5^2q%FP*Z27vG1U7gmVTRNt4 zVeK7(vE=CT#Vx%~2~CCLvO@rZ3;>In86Z%PtOI3b;Lt$`Yr;FA($Q8Wrl9G}8mP`- z0+)>ouR6>;2FUsIYB>kT=WBYeM{HYr1?L$aA$Wt2SbmPzQT+XWC}9%|c$&OWRjL#z zk@W@GdQw#lnHU2LJt=+dhI9O|Lf*Q%8N3=Yhmk}#dfS9@T$?aH_bx`PZQ<^5ZslwM zwJVIBEvCCc`I~L8!ETWg7c7wV5!KQ-h-7PJU`239epdcsJviw}@yjwtiKzmvO z41(BE39ASUL1*QUXf-8>`aot0JqY8UnTG*2tS*t&(Bp4kmBNUtW@lsE0gN{W^;io~ko7VVG?7prQG+4I_{PRcp7wDfq<7d! zQ<~2W_RiZN-$uRD_*-g zG`*S<&I^AZv#}tKoj5eHhmqxNb(Dy*G9+?#=F_C$e3~5@Pm3Zzxg-5N!n1@xCSUm* zq#x-HL15p`wc_bvLEC&K&5EYD^FWsh^!5I%)kVksm~{1GIP>kR-ttJT8+e>YWW>Tl ze1v(|8X)-X)1chz%50hdrmwZ*2SX}0>L^P&k}r`Hti5jDoU5`6$#=^hv%VnWyjiB& zY>fo+^N87Mb+Dtxs7E>E;is`_WO__#~!#2%OUH|t&Er>U{YAe{6V zkRg2_y34bdHwqJx9Wp=U{6(_yNdCteK>3-?eaz)omi)9CFcT=oA6Bmp#GV;VT&U6I z+?!_n7PKs$bF&)Uln5mzJfFjYXKg`pft_aFjw)l$OO{Rb z#<`5#^*Cc7<;9AZ7A}@m!j4T8}7t%_xa(8UXg-^=ZcDz@gr&5ADAET%NOjTzs#00FiJW(u5t+*O(xU8_nOsM=bn{pIiJ3uOq6! z7byP3`^h<=ox(CaQ~45av$!x#{x4f-`%_E}n4 z0UZml-NPaA?54()sz9IqG3)c3i6}HRYdhfSCaXO?sD+2lg|o!FEUOcbJ+hpQ+BN-? zQ%YQ}dTI0+49ZO_ikgsuK-L0RQ2XL>Q2U&*i^W@*vh==S4Z-`$uR)s~(l+uOM{%&h zopOaj-EJzbWA5YCj}&0{6QfrJ^P!Ge7b6$K`KZwrY8G%qPi*}K1Fe9D;Nx?dl2|PAQUGz0Y}jD7R_6LJ#0uLoH)dv0%9%3Y$l(1J>}m*q?}Iax}yayPqhL1w;6$ z$Da46ZCaZt;NEunE<0$xL4xxm?9k?PEDiK&0}XUm-or6PB>DUhi$5o}+;%+W<30zv zkr5s(pS0*)*a7nbV)Vr7dVvevACD!Y?%tX=Gd5o3xED zdWio4c^)E(OMqxB=i_zcllX%9w|ESv7Z{=Yg4*9)2gx7TkZf-oTSrOy7f}h;e&B#; zpB=jI8S_2Kn+U1@Mm@>Q?)~+q5Ubo<&3;YTQ^JWc@M4c$XUq=OJwpF8tC@0*P|oz; zYr?xsHHO{VXr-oUBPUsb*cuT|RI5*z{s^kyM@BAbTz z`=%`Ll(Vr`HiSvDBL- z-8kc&YHDZit}5jphTa^nZh5p*cf4)ckus2ng5xQ4cdFM7o8MmU{n+0bI2VDaX9jxqDquf6iA!ukDa2xJ%@6AZ*T_Ek)16VHlg3Sm7=CTmp zg#9-g+9JH3KkBPNp#9-IfogEOe|6b9OPtwB-;t0dHKRmv)Mpde zRT>dF$8|*MYp)*023YgAuf=OzR2-o%J$wq>F^%H_N#1-lwqE)jW!>B|lFx5~zuu*B ziHv-6-K8~b^H(;%upKIZR9?Z!=d)yrbCO6S8^2J%OTJnSExE7LG%i&|0r-L#Ur_XF z{D3|SxoPE(m)f(&W9t3X>EZPl1Q{#r8(0QvZr~drk2ZdBc;WFm*7&lKB#a4XNX>%B z4KaHg0=mk8OVBYEufMV^Wlg_hPav$@T9)EK0OxyGmnS2?nIV6%;P~qNGd@`bqz3J2 z^_v~l_tRjmXmIvV^r?uV81vWf*r=Q1k(>#Y_Jm{1Mxe*}Rz=8VRkJzr4Wfm=&8)Q> zjqpQ1f7M&&bOzZZEsef-@BU@*PryJJe847Oi0O6zp@rYq{X>g#8)KWwV;&w`t%|Wd z`3rp#`M?&9_Wo=rY37xsGWN}@-fajRG$7^ouBSH%cY=E)|42=P#~0qQVL!A;vY{3t zCNQMy0DPAI%jyu(-Wlrwf9J~|8(;cII@cB=;&A44*8W*EZfHZSIeWD^H4*s?xT|t= zf)a94DC_a%*9kjG6C5fcWeJxB_?E|>)vy;HDd5)iVV!2m{ZOEvY-=2}k&dH)t9$fIL+OxH{?!7EjH^RI{HT zNbc~(dy-M4{GQ}A7SV*_%YIz9k&ju7_#ydNhqO4l>HAYDdphBg=go2R|43(L|Na9! zBER@Ub-T1Z2SrWjbO9)vUR@n}2Z?p#T)*BMfF8G*{VZbk7HN2yB(jQP1SED-h9MTkRW9_;5Yo=Nx*P_~EM4YE{v2k$u zRzwQcY6IZE*H=5>tQLZTTK+%my?IoVS++M^eY>mr-YSQ3W)cKZMp2{y84|h#RAvzZ z1PBy{p&$ebB0&fNyKMz!Qf3GwiUN{^M1}-I7^+GoA|WY25=ek_2_`}a6odpx7~Ti0 z?yL9q{Z`kyYklAP)_U^~8Xlf=&OUqp_IUQ0!H67SU~uD70c4F-&mlhCp99;ZQCEGymw(!} z+Q0Aq(9ih)>UB2*Tr?%;E{CxSFU2|SrMkX;lCmFp&S|)}jniD>(j#m3>IEvrK5~gJ z>ffGT`jJ5~2b%k!#U_U42Y$zkKJe-F;m?g4qV8UP3FUv)TP(E#KL=7_YM91 zz81lbV;qA|^Y--{&*rhM%Bl=rR#^f;yCWYFa#U&XGV0JpZ{PrNR5fr+cJft0UE-do z0l})FJJ8wmL;kz>qs<{-3h`QU5pd+Y9QG$Ww}BuP&~mVAzL?z##2P*d!G2;IkJuB1 z-1E22#%3Z=Hmo{$T4_>ij>T@7Qjb66;_U}#Iw6ruB&s?{VFW*?3PS%mwhc%{U;i-E zb1_0Y*H~R(S71gv5Kc1rYVJWIrZ~K=CU(kFJX{SXWa8g2_ zLXn=!CJ(D2ub?3=npiqNeyEq+4Xi}^5i#F}eAm9k#n85#7WyxVLt$&067RiJZ~g9! zHT=?C|G4!^ro-@ZSN&ZL^;gzYp8>(o4-JZo_Y;4ZXekjS#0(m|yue1l+}9Gq*d}Xp@Xxlb@j{(1? zi61?dxt84b8T?ykCFBCsJ_a_kJ(lT*io<{OnZe66*NDHFf-94Q^`mCqrPOV=%Y5{q zL3lBu#QTp8~*h8mA3=TokjP1u*I_C zQAs+d^&GI%ALfKVDOF`uUB*|PeR+nS_`9gcGVrH=84Pc_Ly>^-+5Mp| zIF;S~X$HwN@ayOhucYYxUz7>*sR|RD>PX&boWL}{QM?9KwH;sqD&**g%#R|POoyQ3 zz;;0Ek|2mG@5F5b4~c~X_O{*upaSB(miVz_Q=#Jtmzc=xiFq&Q=cgt6NNX-H(^&L5 z%aQ#TfhIzLyYyia?Hjj%RcT{6^x2`Q(C9(Q>DvzFG&JqglI>?cV)dXFuxlE2@27tz znNQ}@51ptFZM)inoTP<3ywmDKAaw#|HXkK6@gb^vIK+D^&$a zrwLJqeCS?03c|6tdH#7Y5J~+=PXaXj6Ohx}s$$%(004e|vixDL=KntMzJTU`MThGX zuWdKi>f1AZWEDPQ|JmRrj~{(prIq@=a8dpK%>A{UHWszj`&}va=ku7KZf(T1Cb^;^ zTv1P}-0xc!hAU0lijI9k9y+17x#;)#r-l6m{{~&~D83 zFEsKl{XBKrYJThN*0q%2!3N)755|9UF<>>e1bkyVomWVH`=5jSU~=G37DuPVTZ4># z;qk#BL#F>2WUl#GmR6mfa(j?dL3L+exSmb`%!*I0b+tY8dEo!2tyCir?mr*S*8Ypi>WoXs;+}|q1A3e48i(dzM zoLKVtQN-zl?Ui_)mi*^R!lBduIY<>S;NQuGZSX%bW;8Xty%LvF!nww4Tg(0A8a5X1 z^I+Lzn|9iir4Rms-gt2;Bz8yvBc z{bLnnv+urgcbeOp_b1oNE&*`82E_LP595*IAJ5;rxBQR&>!$%@{kr4#oS8YvYf(T4 z2icE--H5$=#&$DadCi|N(7vM$o_=&?0J9mfXuVaF2#H>Mu+?XN^z+sCYrA&9^nli_ zoE?^R*^dlbHbNJ7`S`n-7#J8}iWUpceiGCodZJ1z+A2RJA1^j|IfMXyWzbS>T)Y1+ zws^?df2(+K&zO@YU-il3$T~g8?Xtw=B@Sj|=+7l#`?kh>p6-=HqPej-C^b1u0kz1(;Ex^8_HxfvNM{r&evrJ28&ZwCPH<0Z?ZfXLMF7;FWfSP95u0i+g zsXSVTnYG2IrED)@5A>|N2xS5+>+AX}iDsan16yC;jR0z^zE_oG{-R@Y;ImsfFXl?- z8|(g<^?pW|2tzBOw~bvR@0d&o-FFAPpLzw5bAJp5NO9i%#@7Z0ryc4raWg*1?>M_%Kyu|LNn_xYF}L;sOR62fJ#N#@FpLDD?(~K z$RS(cuI~OWd@mH4oVf9<@i{zFqfb_jXv*394c6W1YC*~8*8zS`11WyaUVq-C?n&&6(T8eo~n@&6i^jQ|enWIe!jW6OZt zj%D?k)asy<0n;IfKC~NuwC=|vfHc$$foRKr^La0jvwkYP2OzaOPS*jQ)P9HyO!ocW z&%WMfi`j=CdNKJN>*f+^_ZvqByE;_cW={m)@3~WG$faDgay{J%zzD5l5Y@NEJ9aqx^&(BoE_DSv#oVYs2D=XMX zIeqF|R0)>mH|wx0>#0F+-Ur?6?75`3x?0TlK%(t@p@0yLmAQtpx)@ zvtv$1dNCPYpoVwBu@-gksfAW?joD#`yW9wYV~m2&A=~v5_N{90Du*(RE_*Hp&vZJj zuC8j8ibd@Mj&+vj9MZxWYf|TeUm@7;lIf1Ud+v^zO!#^1-gwVV55aW{jQzg1UmWr4 zXfM?zqmKouzhs)NUYAh}uHEXB%CcvV!H{oao%~P6I%O;jClHn^pStKU^RiW|j<0(j z3_xE$xfGJ7YAeR^#Kc~WJ|VuIYmKi6#aYy$)DIUnDy(@K97|3&-xP>F*!rkA$2@th zue=|E2o$;pCPn~`UC389uS#XsTm`H>V|=*R`h?7qCJ7lD37iZtK0NHr?CC~vi&wNa zyIyP&lXhGt?@(S6k~fTQb;r=`G~t<{v4`a;RlEdDJXW&*IJ}R`wc5jaE{dUn>@4=m zI71_;n0XpQ+WzCOOy z=EA91>Z@pDS|EhI*z(@7_O1&dv0xYiXg+AUd>GNVc_9|0rrRMmutV)#L$tYI=1_)!CC)0` z=KHU&2`Pnu1pw{|2Xz4P zf~wRFl_6s&)WaQRoZp1?oOz-FCp2ddRgn#_y=UCif|UGbsaClF$?a;JtebL2ma2Bb zlZV4&hIE-qgbaLLdqI>T1AdX7_?5LdR5TNWQ_Lk z#CCpWR}GkT5|fvdkib(&9UACEmbBQy6hY@v8Kb-)ksHHXL$wyvl!oR#xZ;Tq&wDq$dH}D=JlvK5YXHPo9+Vg)&+=cQ|Z#$P(K!XXhdSE{zzP z&*CY*T?yAhV373#4cK_&!{H8`aWecUgEA$oA0Ra?<0}orhli463?Bpu8~;`-1|#!e zy@~G9t#5L}#Mi-Nnq2W~qo<_rqUu0}LzZbOop4nCQN~a+mLfBh!JoEGW$*vHb@{KT zvX?+-M)7VgQ@2{8+W+}uBLh4WaA}H6+&I-KjYy?%oz-i9zN&r)UZ)jHF`Smu+^fu@_5<#$wqn9?SbtiP z)X#G;v#TT)&Q=`Gnw)G=PLtcdxU4#uG!5TLqqYz00!kE8$L-*AVv3ZVwKZ?ymw6Zj zZ&Vf514hl^(gZKLw>I%gRhtb8yAo%TmUaJ%r$ejwj#(Gp$W}V)R^N4AHGdtQT!#W0 zvsM8lfvt>_X}4{TN6Q9crJ9YE7@0~kWj@^#6D##0wuT%sGU_?!T}+w!3RPefJHEx81EX4gGuCIiwR z3re>-uKNAqj#T~~Gx7deHI1^YIdys;Di17gzUivQ$LOxw-M_+adlq_J9%}~ck1DVf za7$HYb=a_(ick@{q0WZ@>!YzQG4Z*1aJ);FWiQYs0KaYt>?K3lK^txmh3bUcA1B1{ zEwNVI;AzQUhb$RG(*E(IqzdN*L5i&PU|pMpuwuC&n0ANbV``$0scxa=Ak7Y7oZbqf zL@Sl?VXHo}HED|oT&)<&SCVnE^({{+HG1H9BNUf%oM%u}m{{B9O_S%?BxA#IWudWo z&cOMYZD;OZvn2Q{`kUnCw5xc44wzuevy=s$IK5x^hJpecxn(MFBSwZURublS=S!Bi zc=-HGumvhlM)l#P1qj@y*rj1qFYv-epou3#S@lNNJHEtsp9$`dGVC3q^&(;LrWzlI zx^5vNz@3mOho1%LHlwJTURP8)KjbO~_>W%AJFcUn4xnCmVG_mI>>*27-}WiaL7qwcNAwJxcEieMsh1Jz9^%+ zl@|@UW%=x?oZ;mc+)#zk>608@;xEz9ZV1|@RiXX*K#9x0?2R&ur0P3a0V+pFt|IM8 z*UXL22g5{vxjtTtl^{Ueap#$e&IzqL1cPl!?*UVX&Y3CG(5g@obc#)r1mPfTeOqKktJv-YBV$}n z*JOE@^g8-52fZf~!eKH=G;F0)1m&mj6y!=ZnGWQ3IH9o|1{ATkk{3?#3Awwy^szpw zNB+=w93{z+koqDVQD~Nh3|>MEr>?hyev@Q`Hva7!{NH(^_=^LA-J7UEv|+EKX6hEA zb~Y(HuGE1hmD1JV_BW_XatftwqhAL;$usynPw)iNY6KOEWWo@3J_6*M%jT;aIaBI9 z8X7EnW```^XsAo-K!Fn<2Xm}W;?4=>X4nL_p*G*2C<`m&LNSsFs?pm_E!xN zSz2s%n8~*E)>hZ{4OF2BZTyxio`K1-=4qL7%&9-Ll@8^%a)S_(;s#LrI2ke8E}|$x zBFABU!n{ojK<8w78z0^Wv=pC40yGJ31y{!GYE^!lAka`#Tum4hMI(c=nqJ-;DohZP zX|mzW1RiZc;8e%%z+0mTBLNPry_IGIcJC3hUlD6o#7*T1%*|F6|30`tUKG2tc%f4l zzzcVJc)H=+!a>=j1DW>UARJ}~blAn<-CPt_H^Iluzd{)tsLE^hS?YF{g&v%7DB{8g z8SS|ms=OwwckBj92cF5P?NcxcT8p^)B?Tr?yef2)S6~?}dN7c})p{ejk`FLPnm{JW z+Fge_-Z*1xTbVi;*fs)=!A}D@lW0-emBZ!zg#_De^z06aq)L{+OU*ads$=^~dmRy+ zw56_t^-Vt*uG(!$46U-=R@%Yo*!3Yj0+}4yd#nH&Xd2&knBo?ws`hgrxGmA0GnzpBSVY)xV&>e#fl&3k zwBnV0fNma+$v5hIE|M#%=mFNiMHR`zlQ@~wNjB=(+FLChjwU2cV>ulO*8HnQV99j! zfLUT9FD#bZv!4Fnkuy$g=QbH8LLGF&VAX5}*N%otY#+-61$IzmY%A(-MNIiO9X`Jz zX3e34Z7e(P)1`5P8=o`MbmvY68{Va+JgUKf#pn=VmjJjjR`jqf&nax^a!ih!n=r+o1 z%U7JFRXjsIzY1##RQi)QhQkt*Byf<484t&MFmG<7mh4R&7KNSSOmX?K3rFsS9L;P#138 zURMvDQG>q%>e`mmz895dQ0|kO zlu7$?mk*oJw2^d)ZdM}Tb%Bv}aCf9vNdaKiAC*nLUZ|T=g%K`LNTRf%7;+s@(>C0; z>bVywac3>NiD>4qg~#O8XMin6i{sjy>cPf4c9d`&W%B26x;~@AZ$@Je)az!l-d)*+ zx#nA5YpIYjT(z{TW4esS8T8&*c=~KwbZfjmOgNq?%KEP{rS`Yi-?{x=cAXp-v|{*A zM|`eJ~bSzDXN(rdFzZtrV zzOv)5Rq~j+SD6&VuB5QWt_zffAP?+cDH~IWI`mS>6E(y11VC^D*TFe+K}0y#??kgco-{?{OIE;xtq5Mb`{md_6u$o{x^Y%m}* zl&_(#_aW#n7`4B6x~>df(d;L0EYa}MLmaZS>$au$P%61N(*x;QBo$_PsR~jA8wKp( z1++j4v#(E7g);9Hngs^)t5&B~?5-18P3Xf%k*yR6a>?>})(UhXO&Lc4%#!*pNLWtT~g3PYy`Bp((4DSpom*Vzt zp<@~~zn8TP1t{q$ZBJZE^R$8%L#O7f3LUNa3~tbauC*s%e)7NnQ#VvZHt%mk+b;er zy?C3DEH$0&`{EBg*o8LsvlN1Hm&Iu_XGS5rc>UeR7zq?+SMkKnU91BQLtEHyEdSKdr25Iv9r=!!o>1@XQNPo!c4 zld;@Nqq=5q@nnPOOv2?B{P12taj;(1BX-?Y>Le$HZ3I7D$Plb6fXnE{jv>`faO?%o@1e zN94jRUYvftAOj2JfT`azXF0h@bHm_h!Hr#YaF^&PBk~N?CV;b>`%I0sWvpNN{#b5=(k^!^t(kys2K-BM!xfmhGypRg&U)`*e_zIMsGnExNhp( zSEBa*)40=sdqT(;uf=^Ab+1U2wU1&#RsgRWdhVwGDZhO{@f2g_4d9;~2Ix}!qNONg z8EgG8YhSW@J@ezV4{?W;@Z#gIp4FP(MS6ujq`jYz+*)eYu=Wmm?7rvRsoRLXrY)QP^rSvPWU6(w45&b$-51W){5P$`Ay3&t1@D4**_4bIJ+=pf12zrufKtz9VgwJci2hzo= z>f)7hZsU4zxhvJ<;a3aFYPsTsqLM%$9NL6okkPT!;nSC@!V@&5q3*HRAIa7)t?p7i zsN=?l#BI~vyIMzd*`y^D&c_YjWU8Z3|)WB00S z4UcG?4GbKlrKbEo_8#OOMnp3G_kf47S_cxZX*_J~B6VmpiNCvbsHFdv)`64H+6*5( z`=-|A;hzOeA{vtV`%gjKbbb|_&+JKOX^1KiO$GM#f#B=+tCx~Ml2e8bx2+4Cg(!_@-FVBTjyFn(=&8MAO0pL# zwmiFB%z9}9EvIiU@!AgM%8c~Wyuf**5W!}whhjS?S^d(ViyX;^*wxtm z$EO|*&5zut4IiFhzbqiVn{C_Z-tCHj#%o{l{oC@%X7_k-k`~by;uuNYj3>taR4e^0!l@GhLTn)@+R=ZqurKWh2^Gxe9S2_Dj_hx zQlhm9X45LIQg@a`&uK+wLnWplsa1tG<`f$gPO?1Dxaq+;mIqt8(0y=lVU*!t*gTo( zJYv(2z!%D3?dSvuZgM%4`*qs@DzTV*OhA+CSdGLO0!M^VLCkoH$W%rH(kV#+`F0xO z(N?juurVG5^Y$ZdS29fd*QY0t638!S0KC|v*&hTPi#`}$BNnZkA%uNC9{D0jKk7N+ zq0aIDftDj)YhJ|z6Gsvf-|i9$WD#)%5Os6r68&B3h^WXjA13?ayk&w7-R8V?d$aPX za@L+je;E`WF&Dia8V<>lRk`RvnB(UYdJLfsy+5T!&1D=Nv2+fHuRm83sTX*(^kqjAW0tz&YOhadcI}px3K%WsofA`T~;DOHu9x%GwpncUo}Z zhzJaXRb2-dUijeDNKAa9S_|P6uZPKqFY4SS3vHRwveI@~IAy(Y&pvZBKp~XBtL`6X z_)pzzmRG6|>+N8*uNNwC&n;0%L1s%`4C5mr82GP_;*VOQFS7ZSBe$@V}V-KDp$CF+Ac!LAWB}%_KHGVo{k44 z7JE$`;Sd)|xNanl2pj)|BJj7ZA<;dCly>xjk#{2kbGLwej1H<_Cf2oTdKx-)2K^=K zkvOpCyfFIPatN}l{?dRIfpug-ZwSxw4Q0NJ5NpH=f%T+w8?>0R#JRQly{t&MJkMY9 zYB=HIKs#u&xmk#Y%gF*7(qL?~N-~Yol3Iz2w_XJz#8JAWpb@&Py~Ac?^zL{9c_5;g z^)1(o#7Qe| z>~YrUII5-Dy3)g8p;`3jw>?M)n`l~hQv67F>KeY8l=QS7VP&|YEq^3%X^CUIrvSeq zA+4svDE)3eA7jOK2;Ht`%f17R=;4n-w##f9(!HA!)&`DgGxfXNTJYM>{A|A!;yNJ0 z4Aj-x-js)3cOChe7ErEMbT&**o;7nIGTta~v#l@*1NY0=I75d{JprNT9qa%S{w6%rg1}`pSwW1Q2LfsoxTI=23ByZvTY3ja+=0kLs?hoAsyS!_ zS&|-|4drVfE+jfZQ!H>~UqkVo=SecbK$DkEJI7Cg<3=vd#;6{{cL2X$Y%9sg0Op~KE<1dVf#iEuNEAWLRpMuHgYz2N8kEiiC(2- z?0950to^M6Nnq~9!<=Y|+bmiRWzHFkD-=pwTW+UQo4UP{oUSW{%57FFI-_Oq7A@+_ z@${&)&gSLnVc(67`1DC=3VXc6zmMgQI3Trr3++GF*ULL1g*7xz%V)4_=UU|5s6u{B-w@r{1jT3Ray~Z$c?yIBRY;ks&(ny zrhfkxG#=WW37@fRc*>>?)MKMqSmaF?r|WpyoL6EGG=<~QIjTI+cntbR=e=likrrcU~$qp>&D`;!o%9FvMddh z1$1p(>2OT2LjF2(xb`oNks}+XtJM;5=6=2fNg1e#gEMbF9TyM9Wo^RqzHZ~HfeFt) zt_Kk%jgI_I8$V6V+?(#R;Ajh|Lm@e)AWau8bwcmLMGnh3eAyG~z#YfvL;`9Cx}d*B zl;RV11V_nF^W1cCnvPWKR`Y|N@y!wjYj9lAcqxAZQ!qAa9}VeA+vu879#`ecrBG2b zt$*uBP`3F(KlVmvqA8R+m15c@DTctKL8zDfdtDsnV+$kD$aYyc{Yy5rsnm z*#4?&nB;G{;5q9IJQlCi3Y7A3&Nw~P83cFq5#%`qu5i2&sE3hQYDTyUf}Y8 zFm8l5)PZ{ek62?)gh$Oar&h>0g0x5wYOazsY8PMR#dQeppX#=8UaEj&mAC0>1=-EZ zi>D1qyzk6qokHOLs~(V4fEy1qQ@U>6jR+eXpZc~qg(4mXnB|pybQ(s-YBX{lnVi-1 z)`^2F4By{QRVs z(u+L&e7v6WyvXK!rN+_1N{D1nA%i^8 zOj=f~p!gbcnOY7ciWd%_B6DNn2N%ZQZBb;#zP+%490O}Uzfs|M@FuTsv%}w`s#VjRd7s#%+h0+ zgN*|_3*=ORIZV_j7c5t|;^kHD+Y}iaq9tTyqa$G6=^r~Q?~A~5cS+@xrL$%!0VN3)_YYKo1N%KH1B_S+pox zdyJS@3hT)}@L$v6sPMP}E)4W%5NQs}g)Tk+rqv2A1myRHwd5VWZ=RIcxFDYStR-*WCQBL%>n zwEa?nyyDwolxNt3Sxd{Z3~zlpZEBzfVUr4mu2oCAW~#)1(uQT@;boLW=1tiKS<;Eo z{k_K!1`McUo>fC9ZzL^d1-Ke*izW_Gm(|}pWuUNqW$GBV;sUtC@s{s-PX6c`mth5Ha>9RboiisdcJb4YQ+#|yQI3~?q#7k3v8 zliShc)xm`Lgu_B>w$n=Jbs#v9(K|%>VN`GeCZdbbS>OfyMpSg~0hBs_aRKS5931(e4Zgt6Q+(v+l>;vn6C9|(fnz9J;Jd#;4XL!K3H}t^h z+_{$7q%5u5wduNvl?Lv22VOU)yo=cy zrJ%ArZONEDfCFS$`BYWNeCn*_aDh{5jYNza00jWI53mweBv@R%bxJo%?m$8C?b$jc z8~D_&SYaUFlD<={l6rBU*vVAcvupCur{fu@9*B_MQzxIRgfpEQ0ADdfpdbSdrvq!& z@n#!58ySkD4~*073L^awZH7(6fjej!i)SLOl=zYn4(*K0rHbo-VKKT&@52VN+P2(2 zMoK(zrdC!S?M9H%{vD2Jzk(HZOO!sKUBom_lRPw?YxG`GP<>K3E9vhUzo8@N^f>A4 z%gG;~O>QxH$euT5el?pBHEjx#ajhe!k9fHS=83*^+Ze9CCpane3INUfr2G0WOr7up zD(az*^j!qcJ#{*+AK0#K1HfO7_3?f7yt!K7Y|WvZ+w647Gb2w+5(VI09AO*DHSV)u0b)mKn@#wu=PG+0tOx%48d{86JY^`Xfp*ff^8Y zv&47uYfu4-;riqX1v{3yIU3w1`wS$*@Y-Qaq8DZ-F;kfN8A$3_`k#i1E>pac{40IC z0s7|tg+8l8Ok1W44I!f2#^IH-YrCZno5Ef9yczQdI8ZFR+){keMH9sqthcO3j!bPB z9&$aA))@r^b!~E4yq}9rYD*sEd;rpPAfjtz<$Frg^}9-gmy0^tan-PVgO@XraN96j zH+qxyGq`Qm)aa~u@m|N>PD3~`$+^2vmk7obL!_t0r%(vt4y>zyj^Y~*5mlO}d6{JD zMa90@o!Izb^HXn(#kLw!l0@J&7Fpm-oVvNutI`RPWat9ngXrQmsg(v`HG>KWv#kw< zKa9iU-NNe3vhOx**q zh}>gR?O6VkxP6UcD;sX7?6GtN10M4YS64WX96ycTfD3`U0MFRp0vz+23WJyU+BRdA zRx9b#2pk^aL+oC76Ie}nr6aKxl=|=Y#dk$QEGu=uw~bRzj>TfeF`c*KZ2^j-w{jGJ zk9G=x^F??z@N|LXjQ`YJ+-%N7YitEix&pSubvdeqphqdNZgc z4px9jfD`FmsDB3){;(EQaXM5&EWD5h8CzaClA9zESMobwCcSq}t@!*S0!3K)v8k1Z zsRd8h#NwhARu6_U_KG1{=FnCa1;gR&6NIO)R1y?7UE+7F7mtgXOXM?xhaG!nE8ItX zaF!4ufYDa`V>3{~DRR^Z>kc^vzgndyUffrSS=(_ljaFDp?_sRljfDy?AmXyP_F~2^ zA{rOn2ah~+oWzOfQ=QxL)Bx8FRogR8iK%J$hlnc9Pmvzoc(otVxh=zFcpKc&jd2#6^ElIS{pM9;0egA`*F60y|*@k z2f0Jm8k~kI=Q(Mbr|Z(xH;=DaIJZQsMc~JZR$YnUo2W!}A5s3IAi2N}l6AkG`>kzG zCebF_jU5rxW1d;ij1oGQ-&vulv*8aml~0HZlh2aiC|G)ay7(aSv*wd}qT}s{&!J?Fx>A+T^W@wN0!grxKMAWxj>jlIL zYf-kLFAD-m4kKIWo8*R`3qWj;998DjJVNX<_&P}?H4vhvbZ0y4NZsg$I}@tv+29iIo6jH`Y~qz z3^Wq(?j!RL&jAtJXS&g4LFZxU~vBK`yD3Ve%V_<^*m6cP)=ZW zb;**6@L=a3!bqlQX^r!UMX#nA8R69=y)%i*r8iBPPD^2g#^vfjoAv+?buO)PwsnYJ zNMKE)nMK<)w=aFNI*FJ+xBktO8PV{{PiRw?CF`ey4v}3qzD!GyF_t-)zG@P>xhZfV z&Fk2qq-j(_u7jy;tsS(Y-^D*}4&0V54;7)-U0dF*KE_6m*cZTtO;IRK&w-e}l-($l z?l4CmR~5cs$7zD#;dy+_q#sH;FmBydj>^EpggS%6?Jyt2n}4!FRlAn> zz6z`~og}gkV1R28T`yn(&Rh{rhW-DqUQpB`r(^N>AXj?M+=$+1q0Z=&i2yn;E9oe(5*h_ z(nF_J1RV3ZE=t`@r9l33r_5?~%$A9QbEbaJqgOg|TC}n}N9d+sInITa2X)ZlL}}=j zRHR~iftYklGcCA73>^u7Sj3|8FsD+y7NUp}`fTrP5D~Z3jJyyvdwhK>X~Ij`?f3jM zh`O=+Y&=zUJ5u%p`8&5s+cNV*XX8&Vj3;{Y4u<3%Kzrs}2aa2ocX|(QJ_d;|<(Er} z)Qm%@LjQH|e9g-k;KTr;5YWk8Wp-n}%asts8MGHlObM*3;vVON+OW^L5aGB_f?gky z;o94)r;3M6?dX$MZ63b4K?FMkIVLu27-QE?6nU|#F^-U43ie?P%yeWbiHuwh$;ZBu zv=~A*;ZtJd;sUYfLP~taZ2N4?3t;|4ZXAq6zps@#-;t-VPC>R1C+^K=#f>+!%BnYj z9wnpr+1=c`#8cnm8gG-*Z?~jd1$+{4AQE}0(;DrGAb&%4Pb5UdC3t+!Hy8|z8g-(V z!Rrg8j9!mkoRm6i#z~#44)K#_><3V+cdtBT=}9HV(L#386Lvjron)7nZaqZ3#AkMd&$)ZhM*_?IWdO(1-T zJXC2KDNT9hTH=9Jj~5tpN&ViOl31g`*_Jj7?i(Ntvp>|RU=05I+!I8ATkvx|j1)Hi zc+Gtz5pw4z`YoLm9pQ!`Eni`2SfKapU0S$}%|b?o2BMsXij!O%u%pwY!*pm-qr9L^ zIy$eHd{R8Aw5E$ko{X#+l+I=>t?pXdsD2rtViQpbR9B|cP%GN9hb$uroahu8BY%J- zYmKuqB=sV>N%5HwCf(~{2e=CxAY~e&_{(9O6WUTzP520F9>3TWJ|ej% zbI45Kp8}FoK-8T+Fw}-Es7b`^x*w+rE;4vwvx{WDEpZPWU&E4H>jHcHcCYS>+vgr#ML2sTp!TB5Ooc z(l{liEFPDhN=bmDf-c9`6dIHgVWrJdo0zyR8*4O-L`z`lF~xrJPJo_U(QSj48o+UL zEvq|Guwz>$EXhD_&!xa%1Vh>9=fY@Y{Mhl*KOzX?8E~Lpad1*W&&`V!79Qxl5eM33 z#s7L4$43&co?5YsWJhpi#zRanga{YXZs*kmChXIcQjv$&&kIKn-OBO-QOumx%?vL^ z`xn1%Wy@@D1}>hAMUa0wGIUUzePE9c3-H>;#w!>XkGAJ$?XJIVY`{Q15=6@Eqx__Y z`@|fiDN4dWn9`=sjucj5c*8zG&dNGvKu8-4AV6tO^qs`!rHxZ5U4qqVI?`$Ni77}X zTqN2!wgK4$^%jE0X-e{7?qu(@?+XA@EPbQ3VgU7lDo2PkA%vzUNH@FJf-3jMH`6 zTJp615{=Z7&l=c^%z|UZ2y5jiF1>U9>__tTM$@c}%m!s1@V%Y$!uMf1H-(rB+8U}yf`W)xff0rFtaQ&p$5Z0MN&$1o{vvGIg|Mo zh+0I?z)k&;$u}3D(gAs6O^>H3*>*<#Q_q{7iD~|QM1)9vH8&C&4&~aNtDU%Kf1HY_ zz{~-ua+|J7X8lTB0)qf#x-=qDwJzDs{4tptEiqQ5|K*r8$8-oKak70+VtW`PZac^_$3 zkc)+*w3Q3VAMNtM-opN7A41-bFD6+{KuNG){xgcL?@}5#Fc)P4$;HQ6mEoHtX?i0kuqA_>#lhBAFQN#*4=G0 zC4GON>t}G=V$NWLb58 zsF*U-RTbipun^pS3k7f+pk2j7^WiNP9(uNEoV2+*{$(IPW-o5W@GTLA(2nEI>AWNT z(!i*`sRuQvk!ZJ;W;vG#Km+4kuWm>(`XfUd07rrUNS62hC`Fa&--zZWy17UsGY)`2 zjgVG6+D~E~zz(1_>7aMonAA&Q?akH>b6R>3QwYd&VnoWDD% z7&A=5!=+k+l!`d*%UO{L>jHD%!#%n0ArE&vVU9KME&~P5J75?Iy0Pzk((vXfl#aF) zeRr?Ga{wfMbz3<|0ZM2iWs4(eyWwxp=PNtGmI2Hj&?b8G4(9)`_w8Xxrg_88>^swL z$7#y6Jk8k24rEKWMKRuO1W9SwK|e_-8?jd*AL!&nk=A?T<>&-pY%(Y3cB0 z?vP*Z&!EWx=VaFF(0$B$!-&7CM(vm67yBs8hg1p_5b?+IUbjz}9(pkL*X20n5%#w2 zb~eN~gnbAiAqd89%|zj!PQ%!Nx(s0d-a#^ppNpy0Q%)7aQ^!kLI@=hVW}zGKe9{&K zUvJnQ2aPmY428cR?T}LfRNZM13G-$|9$KI3q<$||y$?vjv-QV@kQEn*&%5iw-FT7f z$9xO1E7N<%!`w=HPhHgv3NhU57#uUnc3un>3l0x_pPv72{@}t*^10K8BDKxs=`m>t zM9Tnx?t~z{vE-=&YntqjYTi)pk%%SvX2ymv_Djk**LToNBxXJF*Vg5^-VWLV(wA++ zFy2j)tYt4F;6$nh`*T6Gv%o6&4e?7kkK$6QtIUTEg~R8yZB_ZpIa*fCS<}l z&wH!bowxxw{PFmZx7UQc3gXAilSp5!wlGM!o;QP zGWNUih7f^HxfT@>(DB6zI0o@)>ip1wOJ#<)G&6M=PRikeVYfTq_F^5i%{8^QXR@8H zM!NE|VS*T@0qf{(R4BLGhMPr_p~w))p{bkY<35{JnI2v0enQ!xc}^*`DzLsZXg??! z8G+_%q89=d1t8)ki0F}gSyP#)ypDlBNq9(xwc^G(cV9^v%O)pspj3|HSAOqQME&-f zE0W_zH!aH?wRGBA7-cn?DGJJhq1w0=^L<|KW0bV{`W(&t0I*kZyG<)cVn=_I`c-j% z#DKjU;F~ z(plR34=+5Jl^tQo-aWR@4Ks22{T8q|ac4Iu1>(DHICgL+OfS3i^gjC=QDA<`?Le&O z?7EFX=H%7HhFi=6Vze;>t&l8LGLVjVe{;RLYwplMl3qv)@6#WNPm(57id~|OH%T06 zWJN{~L3kT!JVeViQMOa6hU`j2-d-};T)laqb5@2A4w@7UfM|{d_4O_KdoJf$Udsl^ zUt)9f2rMM%;g)D@d7s9x{S{h_S#XEv%*Bpyc%mh3x3o=`Bd#0q;qEmTHxzg~cW!P0 z-OMsTr0m$CQ-R8X+TFfcu~?JthjN2ipQfFF$GO4~ny>Uxd)tiEO~&KyC-=g*6^Kc^ zx~xdqx~C*K80*Q{mT$#u%@k(80CCq3=M-PLh;CLK4wl+ zQn%8l)_5Vlv)pzI%MGRd!b*IyY>J%j>$s?U%Aen=8`G67LnN&2KQ=HMsv`oWA}R9$ z(VUaP*GlcE0IPQmiZ#C+9x?4U%adKFL}0z@hP0Uj-md9Cw1Uk~mHY+jNV_tU<53I@ zsOjSFeN!u&2prgwiJ@949cOf%lAE5j9YN zn+jYN175?O@B4Cwt!wTcnvzyu6_&XnH)og9PG_IfE9F{0kZSL%F0bebE0-L*zgQPa zZZl^$J=dBA?z2;5U%rBUbJ*w3w`SK0SL|Hv{oo4>S8;$5&|9?{R>X;P!bd{dP!+wF zT%g@^xetK97^ax>+kThedytjQRye*cM5({|Yb zP;e1xILsr1O^EIp?uTF!2Zyjh*;rwcKMc4@=>_O{&CJ2yq#^Qfk9Lw_pqUnt@MFK* zoq@2$Z|v79*+t}swfb__pNGwNz#0d!a6ebkH6=BmK#qUCmQr$b>%6shwQ-M z^;z3i6QmX0rFvh>MEB05iTYumJ0`i;L}-!sPN zJ{a|vdBimFh_02ww+%DlwHRs%%#W>VC^7g7B$$82xzXnvSzHsA*tSUUEM#oIo3J}P zetPOu8iJRghm-U+R%0dBG`%wwLaKl*TAVG7?x>V7MEx_r%R6n^*Lit7y%`hNHo3i? zK(SS&_gh+^6O^a%=-$fqZ3v$5L9YS6_t6o$*I5+L3MiqLmXI1rS=f{bP(S>QQEl^! zmBkNG;gCT7hp;MOH#lD)mDz$>(x7nw-d)yy+oafcg4zyG>4J>M*YR^hP1kbJf+gYuR zY3C(8H*lCQp&H?iNxe-sy7yilVnWC@_MgCd?6z1nN>$7cexE4djcy| zrH%UX3mnl$Ba9$GD9XI<#$r)H7WYhG2kG74XhDo-8)kr5oV;0R3fqR)~&YQ_lrnvR+*nDJ_+?h1<~6(fdg z1p3)zI&m6LjQV=93X$Z5&Y}kHyJu+hSA_lA{!^jSP(&Ie;q(5Nv8bF4ej{=C=m0@2 z;LaBnu7Dz$bi%DrE9biSL*!HnFJd(;|7*d;JNua+$pmy|&;DQ#5a~rRznNO&D!l67 zbWvcj+;tP5+|ZKTQsJy@?fS&=lDh|nkr5bkMD;K>oIHf%-)m8`_*^q-?|AtM+mB#hGsb}+H4biwf*|bIKlPtn?zA-&=)u} zPqqtsp)a}yHEt;(j+hc!f@tFq$uJfz^Vq0GK$>d?!ArS)R&n>jZ$hgmaV^p$foFs- z!kcY06)F;&Xa*I+kXQU+b&&&shPZi{Kp(=t^#be2C^iS1380Ytr*SVkRz!$IBF?5q z>@nkJMu6lxYh*-tHEgDF2z3<+qKmg3Lt|(mv{i_pm*jF5jJLNJz9rv^HxOFai)0Du zGA|JRj9%Ho&4VE;iO}$vZu5Mv0L#4J#Qj~>-JR5t$XmERBaN`sHq6RWu69hyla}So zojqn68KN{D%izMx1H9`;u(pj?V9Mxt=hBd}}Ub8~>^}yB0LgY>u$06egqoV8P z5T6a|QEr`J22E^oL!z0~aYA6tuB9Cx7AMwIWQ#UdoUl=L+8DBvc6X8W+@tHiBV|m_ zmchjt)OX7|%(g?YYs^z3tYS~ELI}J{L{Uz5BU_4rQwZ{%yyW_v^~EqG{>R#(Cv`)S zwm2~&W`#-;DWvjinDEz`{D<#NG|zBn;DQ!x#hi8JyKby=&L)z7F`wISJ|g<5|E1dA z01Q`Xc2b%+n9?XulD{t3tP9Lt$AfR=Ld}#oLYWs6M<@+Dz@}x6M5HP3X62`h1Pl22 zRY*MMYdz{Wh?W*_>CHf7jIOKM*gm{2P@3Oo(Y{AFutnO}bT}aduI~9&G}3SYW9s<5 z09{XzcepxR7UW#l0xHrvQ!&Q8Ua@IX^vd(8zA{SVP6QKL>F=9UaTx0o0nIP#7|=#n z`RfSjuERC71RV8eH+V+y_@;w1nuzqTMV=6Vzn+kKOEP4BWgO zq}M;ntQRk@H#??&Ayxg#=z*%as551kg39Q+h=hx@9^SDBQb8X78M(HF5wR&;6T!sS zBpH|NC3hz}-4A4P4dUX%;}O(lj&4CO>lBDzBm`8g5srjTJFX8mNb9#QLCW8Rtt%DV zlYQN?X)9n_8(rwqGuufIgP6KCC0lz17p|IWWwE1mHms%8ybrJpJ*mY$sWw=jQjI%M zvq(ovhPGEzbPZ0VGPZkb_QEbi-Q@2v&vmM5Gi@J(l$>L4vMLF5c1@ev596VHg5qy3 zd6+v9pRFs8y$fNt0Phkz>~pI5IytF`H=y4jT;qyrYia2YvfRoNb;s|HhRm?iNg!oC zz#Tvn4nqlLn`~0#DBeEu*`-tt^B=rh2V=Nj7CDhlWMGGzE-V~_4pkE8?PGWzz%)lY`$}3fBS0{#hjVhGAT8WCr#N8Z)T(8upd0k)BXp1; zWfB9*6FYrrLnuw|OQB{v4!E4%rq5B0{eAT#BOjL+vpbKi|KSWWY{=u-3D{#(vs@gY z7RM6Rd=k=pwY(IX+ldOEz;UpAGX}ZA+0umP)O)A4*cNAbl$7qqt>6>`pNF-Z(s;*< z*(AT+$Sbo>*irE5 z>=$41$5vHbh~wtqEq?_k+R*mSSkGSMvqPAO60eeLi#`fB^y;4??64X+deO2<}tO z%bFlnlVd@?6`zEJpjwN2BQ#4k?GlT8vaojW%cY`LzL};K2ib*b-Q#C=jgyuQ6Vk6B z@QU05hsgCH(`xyY{Oz@#$? zwMiPrnOO=z?MBQ#mAA{j{Nzysq zKjJ06cbt^p-e<`rrOmsC+gLWS-1&MjgrA>_2&_b$;C&#Ay(QUvb#9fH9Yrv^{*(5L z&N8WBeE7r`tzFLI{k!!mwH6zUZlJNRT{k7VI8W@j;xltKHQa2e2VnF0uW=cuXvj;5b*|Jxg)4J>(FZV$sJGZr6LH5L3Np3m@chJQ5#&^xmy6Z4+d8M41 zh}6j~aKzf`24%(Zo=#IRx^;sf_5I>2u~Rv0OS%TmET8)jmEzw-e!VFG#XNNgYE+XR zUxP|TIuPSPuE!z&rlVEi9ECUcHN$ch$n!bR=CgSb=OnO=izOtVN7q}J(YHQTv}Tg< z(!ce%1QXKww?toy)nzEGm3_K`s=HWkXH&l}_|3)`dpPO}@pFc>rj5&TPy!cj zFIo3a6lPgd!JC`)pA#W*A?aFs%@T*E?d`ZAGY!VmS!EDi?oNz;gsW@QJ8taF`M*+k zC-R2_B+OCo8+%Uty(c-+;n=h5&2-eTG4K~DZnV(KILEYA2*jE;`BRS!T}C~(8nqGa zLmKnteE|%hvcRfuKs)K74K{QUgb9ao=A!7N)JPaS11st%=584Ctpl}Jr{%n-VHAvC zVkF#|9L?_J*u{rUJO_uZEC z?DN9pPmkms0o53wMu`lL3;7Uu1H;6Z`Z(d)K3}a(_-m8c;N*kl>p)ZK0Nsau1R9s+ zFVJ)yc2*CwYf01M?<12iyM3u&GfYld6r1+C+%t@8TP%8DGdEZ;NH85UvXX^B#S47!Vd1h9!E{;|z#jEv z9VxK&e%q6#1quO)|<;SjOm})-920iHByw@~jDk2$dkqBNt4jc``7F$IQfsVs2n4Q1TB4g?Q zCp}Z{TQxMUsqgj|C|Z5=ERiO*z}G1+C9;JNk_}j9hy-0*)~9JW3|(r^igX~RA@eEF zDk6El0B@?DNvn{y7fUy0!USv-Qeu2Kay=-o{i!sl2f6_`HAf+S!iW_D1&}Blkp(U0 zzQ7_v@>9zLoXvU_4|HoNg{487=jmaf3L$VXrq?2Wd{?HZOyt4sEx#=pV4XoBOZJ)* z3L?`43X8ZsO!p~Lx<9an1%$wLpz|i9y>DUW)>Epmv1|}DQw4EAj@h%aB||LjmWL!I zxh-xW>ofk74%MBMhyU1l;O&l~R!pb5>q^h_c@dLEY3s9h$`Z=H7G&){?fXg7ok@&})fCX`_Gw8X7I(_RX6w+8;kdaT`E#AOD4)+0FWbgt7 zDtK#0{RNFq-R62Py6==w%cSzwn7H`UogRckR)t&JPTJWwvll$I({~fj^AVzBr$Z- zMF+2{B8oO;c9C>4dBFYG*LVFhbVntuf;4L(^S{8@b6}@hj&0@ZZ0|_WfbcAH%5FQj zP?7mVs60M;l0{a5S=+{Eo*nChordPbHPa~nwu0QVliECS&DDrjoEhZ2 z>~Bq8vt^>|7H1#&>!7=>wDMnMiZnZY6U97n{mf0rZp-abFzHR2&{WoZ9hA^q-^w@m zb2J^%PsTmGQ}gyKiR)E)L&E9?RsUMJiG$I~^Sr|JJjWavm_Pw%Z{zl5ov3S^FDSwc zx!(_{h#nC8pmNf51hj5EOq5Fl65TYSGAKY(V03+Cxp1BgSU&=T?lrcWKp;UsYEy;s zr5E?;Z3i=kL;gSj=z#?NGh>#yJHi$+=Ouu`G2)uBXrpl_P%__VQ#0A0NeZG>u)w$g zk4oM7>7L1sH@s4QrbJ|IbQd$5LDSnO)@&_q31)6yi2!x*md0x{3jXtUYW!Z9-C7VH z?%8nd+ErjC)L)0m@BkM#X%8_WI>niz(9ICOyZTSTMZPcY^Iq;_5=B8T_C;RHmGR$$ zd*8C1Vy{a7SjW}%&c~Ki*ml1Z2|TG257S8$ZIRTa{7m=DXn(=H%b5^l6*dmteLqTG z)b%oE_Ut08SLXv@`CHEue!dA3pqD2M2$(Hed~{pPh{-UgNj%^%gk8lhCiCnHA?a1U zeTc8}y?Bwn=BB1sk`8762{<{+SEV~?3b8TSBO_eOxPGiUTq_VdR?-rZMDoJ zzdfvMk+`C463-@kk+`7273;V~nj)URc?4bI2u6@_#Wm;k{sPN)`cin)JLQ>Q;x90d z#jKq1;!#Gxc~vXfF)cxC|DJ#?E!XU1)^S*OXEX8!&`z$yWLTZJ`OOjU^oOq&9&@TW$q19~*tG9G*`hdb?umxzY}qZC(6fuf*Hl zs-l8CFTOQ*VByM9U(Nu$6*z)|;d9$X_GL$6!V4LS`q0DqPS0JdGhggW z$Gs|Uf7wS#e^%j{9SZ{c`F>P0nn}$eK;2Bxz|6CX&k4o z4GK)H1cS9YyNtbK8XdVk+dz?U0?yZ^xVJnYKL2ivwP;&Q2Wn+BvZ@|q6pf()RaK{O zWNuEkn#~&}Ge9)Qa#YdOSC2{yG}ITw%w99r4`2VMv~sf{l6aub=1a`V&%Shk9R;;e zB@fS3ywm3!-pKPV#EMe<9cgK%WxZibin%e;RamgeO475B&^lleUH@!ysw=G1FjM5i z5qmwQM=Q7StW4=xMP(!uCDd_DqO1&hUX_E4Noxbp+7N8n%xyusd0JL{&ze%jo4HdU z-;q%@77?_M6ata4tA#k$?3Nat-k>oSH<~-27~AO<9^d1@*rt^~{9M#3OpWIattto7NgoweuacLUj2normvX!>`e{;IeP^58&i zwE$1)_GOzvP+dH064Vt@1FUF}Jy{O|dcz5k@u8rDCgX&*Hz}oBegerB>S#qBf)bpV z#{9e&FD(yNT9}b_fH4-ANW_hx9!LX0r3-65waYNyy9#9e{atyR&hz^A#br|>^4Xl? z+wNdmO7y^YAWsv-Ud+>sHHb5UgXI7ju}1xwWP}j3JtD7Eqt+n!8L6p|dxEII)Xf6K zsmUl?yL(4|Ghl{6)Xv6*W(*%)a$CB?KWZfWbqEHGo9xXxkN~#sotc`uE_ob}HOU3( zt)2}oN7cQng<9TO*hpJ8=r-1Gz=*N*0TS{>Hz_cSX930@_2qZkfU;X8Ti#%qMzP=7 z5+%O_wuBuc#n94JDnC+tJJsb&)wg!wLMVv8Vz*<$x1rtSK?C(ga z)q-5HRWFopNfX~{rbC$i!3&4X7|wRxXa9*Owz?*5+rA@@R-uYdfnL@RrSl)zq$6*B z#6G$5g@AQB0ffxYX`OHGi!QjIx7Y0K18I9{TK>4PsuvG*^1+&Q8Jf1TK)B{EvoQ+U zQUGSxxHTKtwy1?slkx8#FntaOt|2#|T9Rz8B^5(*ra0ACc#6iCR~$ru>l zgk0f(5p3>vWB2aiW+Ga;cRjxqsI_q)dBeQP7hraqcg@!bKGZDzS>4&NSjIuS?vVcn zY|O5&scv3#ncS+sd|gbJ`nlOHDMZ@xZJ#?X*XK$%%@#OJNmq^OSh1YcFM}~R-3<+K zu>97W17|}fhG<8iWdkj-MxD!O;)4fn-f@kb#yM3m>pcgb+q%A@arT3b(g zEf+Y0o5gE{=bIa`(=nOMuUfonsZO1{z(}613*|Uc5|T*RAF|sa46wz3=mA&H#vQp? z03t57tT?VZKmyv(YqU7RY7qy+#_ls7ln= zZTdaRfyV-hF47Y8si=;o$jbe#ZHs7&S>BVgW%&!h&BW$`&wI#*KWPQWtainyBoixq z89QT{KqNN^m-j>l#tHV5h2meK#bI|o;sLmqBP9x1wsEc*u%cgaxW_2@7gqzicAFHy+!jX_|d z8!SI?;glbvaZj)N76L*Mb#r`Q_nkrWUUH0fsxNg9i0FV>aiAjmV49Sr8Z6p$|=%P^gWU~tS8^4}jinVBYv z>%*x6$~7?Dpdb{qh3KHirPau$KA)0q^U8aKLug-+nU83sMIcp&9Yz}dZnXY|D%?8} zzGN2YZwHl?s?+R#31QK~J&^H8;(*U>?hv{vICQpXTTJPXogcmStpW z>4DnPk)j%$+JzA1wl4S?nDb-@)E!*>lBGn2!G9x%OQ%k@ z&VVeYszJB(=wg|sj$^O)fLV5M+jc{)e%&6D@v!~C1xdW%4*&hq>hp8mH$4ZR_k`q%l{MX!(H9#@|gl|{f}Et8;|d+5*H>6;8a zX<0tt8n@R$#U$<@H9Y*9rTM558z_=r!i@WK%4!}VclKv0!ZUnf^=38kSx437I-pSg zYhLnEZxm|6^X^vu#mp3gny@8V_)|R4mR8ia+xt1(C5vk zNU2I^b#pFiIkDXEsJ6E-Hh5}YtPQFm((y@qM?jpws&K-fC1oX|B_6%peUZaeXA}KP zUhdy!CnnyYtwJGqjV-LR41~F4ZQA&f zwpSg4_OIGrMF$`aOQp5HDz;2m!nyo=payPvAL~i&o)d3#*xl-{3=dEFAiP75`MH2v zq(}jqw9$$exXRW=rW$qP0o)HT4i@?~t0rij4KMBhDK4W-$M|`-jT^>rW0jz4Q&bkH zaM=310%U1U_T&vuKn&hi5kcn^_ zVt+Qj_*T%{yrAl~A_C6W*?Zu`ZSQMrv5t)c%cL@}5j@guZ^pLmbfQk(rW!APpL31mxEhSl*uA!_!MXPntg7196n?-mpGN!su~ zb2k-I{>`Am-{-FX>nuzaasE|%sI=_=-kVfZ0$XF`o8sE5OA}-Rw)4I8)Y3fuc@F45 z!G8AmpdD}!RW4#yMEM#uhcanX6(H*3U+ZCD3#_%=R~qvE;PC#oT5xdq<`eSQ9}W&5 zxu-f@ydclbeCgoukqLiaaB%oqyyDk`!^4lO4j1n}ipzhnc>D0k$_mjZuaaQ#7dNQx zt@^Xv;i=49p*n9}El2g+$_`JP@24KHetDGsHmQs1c>4eT{Xb#gasLJ3E;xy`h2OMz z!2RMP&c(RDS8o5``$?Z^VdOl4VAdwRun3REKHR5m*s?Xtz`K)7x6d*mo_ebnuLK-~ zebt{OdaZghaqAOK;*|z`KV*zb>qH zY2V$Q*C*cJGt%kTh1D*c{q>%YD@s2stftm|?bmzunf|)4+T*-qzuvR1_}7Ki{tL{1 zDf3@8d#|1TQ<(|F&L?x``nIWL@3@GOCHh{a?JSHwmbj}Zpb15}>;p7kP->`>d&6Hj zO-;?Ypit33eH{UN{l@4F1g6pjX|5j&x&Gk5YnQvTJ8q~hc_05(eANmL)h(RA!Qb&I zxNu?MW0kRv&s`n2>AuSy)fU!Q^s6#Zd-asn>~hz~Wfiz^3Hh4`sm`~WJ%0n5@Flo( zvg0?)A#Z z7Y3N+Du5oZWsRy3vQ~9PEs|ws10da^dhKw=!6x;&GHZkC&82n2+zrsw=Oy2%me9WJ zyYNlB=%wn4+AF)hmXc!?iK+{)9tiq=z0rj?1J*4cXh%EuD;Cu(j-?3!e733uF8D-_ zyfBd4oL_uy3u1_F(+9u|`w~=7_nGQ@YR>nD=e}u=V5(kg*PsFM6@}HRH);hJ&C}m~ z0BOTuhw4X*%c(h6-v!Sak*oSza0f*C%@L)Gm+Fn0b85cByI&~&O81dQ;X#V-rlm^> zcU5SZ0})dz@HTXw5xxdc%OD-*oYQ{nq@g+=j4 zzv+C`Km7|pCzt8az|wV}s~+yszB@c{Qzz8|eWnf{W6sii`v-U?9{!3ewF|{ZF`Gtf zRa~l8H=YE(^b0TSVGUdRZq5f;nF_C_)ViR>r`*HH(x0{IHo+jWS0hgbPpC_5auF>si~pgfY(ZuWWT!S zIrl3Ms8)D9_Hgl=s@F?@1*>m^HC>m4M)lDpP%85aDwe+DoSXN7Nf$qPJob$WjBQJo zRsWW}X#IsHmp**;JszlphNUM}ee~-4pbuzV`bvPU0`SJKi(INYCI5vuOW&~Y{s79- z2mjmu1zLi$kj{Dd<$EV1#gl6Kx!8@mS>^v(c~bpV)}2Fn%x0q{zbJe|Wn9q8iPE^m zaAj0wT$_qp)4W6neN^5d@DF$U%X>BZT;*xNF=?IjmuITfnKo%W_0fBu$+J~jZt}21 zssnDBG1w;Ess+Aa5y;RATY83!s_P@P!L6>(>Ia_0{fo3XS4< zolhdJHznJ;|!~}2eQP&bc4Lukn zStxQeRx9xEL!`?c=|hjiQg0>!Dw;^ebI^Qg_Eg>cT=rCo5{e#fmp~*> z8BeVJu9@)73XK+D#dEZnqp$Y+Hll80 z8{$NC2uj|ru`_bC>|O0j@F?EjKj$R53pId{-la?LYp6NzCWP+kn)+)A!=*vep)BWx z`pBg0Da^$PSz8dE7vkd!smtX_6p&yvSR5aqOP$du#JtIHFJFU5fi4(M&6$V#Q5PJr zm^UnX{T`nzQABb)EkgQMrt1qq%iARb(q2dEDU4=8;Cm5vmTFB0DgfHUXFaL2pFX}i zDq00(|8;fke2KBLt|r6>E*l8&P-T=L3D7|Vk%Y`>a zIStJWq~evrDkeeEw~tAP9qHF%LKO>HYiK1uDF(`EgoUT&TJDOTE-m*VozHUr<*}Bj ztuBed_!T|3aD(ZSvgCQ zD6#Q~!2g# zYEZVgUsMj}18954i8N@~;7{mtQ-ZhKt%XCo3TFn^F&pLmTJG_RW(icgaD&v=T#q`N zhi1OjAe>lkU};rv1oolF8*a~jGD^wyQ%^ER)5VEEF}+&5VUh3Ge|L~Qh%@sjeW zro8ui$-f8Z-D`=@MG?Sb{M!TxG`u7!e2V7+pE|qDS~*98qlMD!5`gy6`{|9!42ICf zPxfqIy3GQ->_1uif)L^kmk*@w(o1%+xTfC@#CzeD%g&;K9X2GRqI%KO>|Cf=;h=sj z{A{!@*lsTw?jH+%il6_BTB2)1W5vf>V#7e}n{oUw{|@}(R1=(W!RC>R$-$e*uJ+n&QTzsZgI zsp_ObkxzWK#IIs*|CCNviUB-!YBS#3eEqR-vp;;Cr_5-MiYRoOFr<34xCIObwc^ej z6vfW4UI3np9&45~Z@(hlAKJa`H~LTQ)dd`r`LV9;{&5YJ_XIzjwQ87O&jbb{zdq_{ z?3bfmd?vOLXuw>kRN-)fZd00+S=IK3AOTP`IFi=5wWH;H_w~mnZ(i1K-{r!o4FeQx zP#&|tQQ-L-9i;a7wHtxJ&-GKYAl5hj{0((;!Dp1D8OrvGiV$c;QaIBNtfa-n#sP^p z#>~poE2!PFO8q0<+J6X^T)$n*9s2z{*bq}3KDsPaMfYQN^8;C`)mj+2ekyd&Nh1pF z+B`@8Q0i2ygb}u6uYo9^l3+lyxT^!`cpyRh9^H_7#g-!gKFA-!rx!<001u}zBzrdw z{2H*%t+_4j`J5QR7+{@+=MF=XL+k5y7L6R8`B-WGj&|DJr`$iay#n*L*`OOe!y4xT zbvc&>noHXO0Yr;<7pWxYKf}B2+B1OefRoEZpmFo9$p-Ft`0U@9da{N1dAg0%Q`zF0 z3*~d5t#T(85>|&rHqja6*SsO{e?AiZ#?gl7J}H;wLgkPDqI?pwV>L(zNh-Qt0OkFi zUG%$%p3uCXe$9mRai}Xm1RJPSMZb_oauL3irZ6_Db8o@NhG@xupez=!}$v!Wh(+lrQotV2y`i4|yN z6z1KGR^wzTUUEBr{7HZ^krN@PQZi_|=m`x%ZUE88ZX=wB;kT#*VjliOVw!VMB8GXW z&-^o?i;p*84xk-EedIsqCjwy{e=euKVH3WY|DH5STVWdoft z95|woKc#1c7WQpWVx%(<1^;jwGSk*zUqR5w0;TnA#F<(`AK84Wj@(d0Xo%+_;6ss| zAqxq$+PoKq#w@_mJjEb-zOtlZK&fXf?lWdRxIXidBG}OwDF>Lu+%^wLXPyBHsjnzY zaj^cDjoJkZ2A}`2{SokUA_sMfnlu9xab>N1qzNCe1_+EIs8AGSb78uXfEj;|QSxfw zal6Y_C4>ED)mQDpMt0Q<4l;e&Z+H!j z;2==RkY$Cqd%6O-3_0{jn$}QA6^{Q*nIg&NmH`yG>ebI&Srf|}rO(sN=aZ*YBEcj; zFu)F26#O!5{PDG01G4@Ih;Ns8IMx9AdQY-J=%9xpVkmx|$c%ea(=eQ>Y+GhDC@FXG zt)1FlbaBt}zarv#I8tl*7l}yQ{6px039^VD&3=cP;In#7YAz4sko@Sf8vFdI zda?~cqp&uX6BWSJI%Xg$Vk>^5V z+Yzwq7v14tPC=n0-8WQ=^|J6tG^c}BUb70>LT(sf{!Et%L!~j%><(Hp`E5XGh#p%u z#fcqf@#@7ynl4`vI0cv!*e@kuvmSTU!zk~fnPcr>*Isz9S|1mm2$kKYPTbAq%qlUj zpV(@^AH923_z3fQKvI9<^#F$>n{uhYFg;L(R>~7JjIr>Buk^z|T}MAvV_cwoJ47w) zM?1@U8WtGrDGTM(Au&hxra2MHr#R4+TKp1QH4pBh>Hm8M&YtzhDyZ#fWA_c1+Y)oC z;D>nmUvekWOxIk_yPm0f@#DCjldwyD8!S$+uMDqV{skV%9~P6lMC#|$y(T+lZee2CJ?t!#VDX+e3xJaPd9q{$-G1$mrOMPV$+~;0^tdK zL;d$6LwsSw43|MOvZqRS`%!g7jukUaq1I@xh@j5g;k6+@V_Tb;!pwI>E3PNM_aC@TP#hKR( z=?epNih<4R(i)CZc~Lr}rJPu1Or5@8x7+}=_Vq5Z&U#%g&@q#W5!y7;N!jBrI-etB zPeLok`ReRsmwyZ>f9eQi$8?_Z9+^Zg!>nSjZW);352NQZrs}_sXEI}G+m8*^#|g&r z(}5v%cPidrw2;Mw&UOy3*;O>sIbFN*nkPhT72lr%7gMOOH*3(9lPV5b0T@O`G1w%X z$A@}iW}n-hA!a1CdLCKte(>vPYx5T2n-mRt{M3-x7$bf%&(ZwsEx*K7?GgLm8i-#( zukLN^mNf%k3T_t-D5ba1mZslsqY&39Gp8PfE9H9iu^bzD_gwP0{L`O{ABRnGqBOL0 zt4qoDGhqy}#bgB%tqcL%V8cK>{D)x$IRXwe$dm%?+u!?;4CCT(`$LR5VG;2%nphzR z%rR&qJAY%D{S9|)W5jU#$NH?iVfzm*k1+ECgj%2kctG|SILR)jQuOa=7&GkcfGhd# zYW8$#{q*5%l?1tm!eiem7`fK(2&zr#mfYszCl^`@{;iu40-X+DJ318aZmu`At7xIy z0T>f;V4tSWUv?jw+sBC$jUH&WN=V=r^2n=fXpQc{LCIeV$qVX3#-Zpl@F*bZI3xmN z?-vH(5}6hRFx)%QN>?kev8>R0Go3eU#0ml?=@f8?O3okW-`zG zL-dBN7s~)N!n_hU3-$UtX!z4#uU)&*WxZP?;QoCz60D<*G_7WV+Ur74yYT9dG#GI7 zvMUHeMa3>jeETv&{q(X%A?q)M-TJZir(O?HC2`$_f48i@>he=?ddks|l)@?LHyxMt zTi?ddMfc^TYYq0s-a0c|XTz*VPI00YO@N$}NW#K59j{;P6_Vu&BTkgCbXVvcXeWw$ z1@|xx0{M_aUF;%d;b zL;Wzj`JeHj3j~{s^Qksc9rrcK22<n~M5e z&H0<9Kuy5U-`UMSLF@UIgA5TFc&r+)*I{cHT56z}u|_>v8h%;A54K;w{_dN(@J~*Y zkiH&|sK{Ca;-J~AX)u2h-v35t+02Ahlfn|io>7Y0G9;T!BCLr`& zQHL>R1e0z@!~<_<3!&q2S4XHcDwH~W$6**BU;{L9tn^+|>soHJBhMZ0(0CH;P4jv& ziYKmP@^2o*>Uv&L+_3A&Xi^wvVaXXZ%-dgWa9* zD9m@(7bVOPz4c#+V-uoci-uj%z=BO|46mDulBu(8;Z^Kj%-~r36g8?b5^YTA%VCi) z_OcSAhRUvQ>mu^2RT~+!VWA~=D1LesT#^{i43*WuWhrqz+b0dDjG-Q*b>JkRF8-oo z0wxPEw8Cx!PkCFk@0BVf_0r56%AqYkZC6uM&K|b=*3FoTnXY#!T))<`#V$h%7+W#D zLPJkFkPcUwY=u`ou}E2B>w=Yxl9j7LdZljI4--D@mVxwtalqUD#xoEI8kAu^xwNk9 zecT0@B`w>JL8FF&n>8Gzaja2*q>jNCa45n&2`+pUM8}9I%myL8yFuLI4w++>C`UBn zTh^#-{dC~1$IIft-n2qnlJZwPvQ&*YxIrmDxwB=V8;=%<_r|J6`JbzaACU6u$mQv& zPwA+HoTe}rcyb}<5&YGh4iR&19m=C7pU)pX(38u4$)1E`UOmU?DQ^i0%EUGGQrAZQ zK0nc6j@VLJsC?%V+g2iP*+o@45$4O`QIjO1PEp>y7f%p=y(F7 zI*ob8qAmmumEc3*){?hEjc!c+&N)?p(lNFnD)#vlz@W0fN<%khaGbbIabNVpL@l7} zJ!_qsnsK@^_#H-n#H>^O*g>;9Nnc8VPmiV=t6eDiC5%**w6{?=#-Qk$OVKr;61kjt zuz$J_DsRD+lR^S@wgN>^qNessYW3#Fmw~X)?KO?u`%%!jT84RCPe2(IxF7Pn&q z<5mRdGX_%uEM~BMz|D1)a#VWbc@cAHejZPiM8NB%viOlGg5tHF41(8FY^DZ@Y~lqP z<&8>7Lp1dt!U_{6#{LR$bSZ}#;?f{iA zsFO5fOmFDCl?wkD4zqrY6iSIz5rIaX?|?>6u_iQ*wSWLlZJ0ekKG=FJ@=t`K+0?l= z%_z7o{I%Sq@b5eHE3?Z|4B}sln^Ahw*5+kO4!^`-t^3xJ&Z;An{Jq;fX6lNsQPDLs z#~npKL4h|nG_^vl;M`ISz5B=fDT;L5pdAe~Gk(#W99ZYB?6px`F~$tnbyFu-P^C@; zxf0VwVJd}Dx6$(Ppl)iMh_R+IME$cH)55~JCmf>mg$CzB$@O>6VJ8=SVt0-XY`2JG zHWk)Mwczs4@C`(F5Y4#H0AbbPbsFv<6yggJ&W$n>siV)wmC|BkZ+Tag+#Ihm+XoSo zq6`^}(>ojgP>>Dp(h1KH?GSM!usFqzJG14MRHW) zorJ#NErYKn9{Z$qSC~CEH>?n$NrkujfGxKAQQdZ?1)275O zC{a0~W$`p_4r~P;Gipt=Nirj&;chtAP!coNOL!)ubs(<}h;{kaTJ9zs3(16(o@RKH z4uC@mMy+pDFPvK9EhN!f*{dWaUIu$uN6NSRYw1rBk(m+f5ptePSMW+W_Kr^Ni!q1o z$}2!b{m-r%vu3*6{fl}ug#_{;$;Paiw*+tfj4}zE-G*8P#l9$Bdk~6)u@{$dFx*w0 zMAgT4bbf)LsJoH)Ig)1c6*u$9i2WbhN{{N3Mj;KP=^gvTg2&QTqD7^(*CT1-XIuE_ zIIe(fQgmktXy$y24E}}`n^X7b^&=4e%$9O7O&Ym0hl>38Fb5p{Z+x{x3kN`Or57g} zfj;MzqF8&&ebFx)4Uoyv=U>T=qdQZM@H;XH^Zlx3vdht}(*mS8Sa5XaEh?s-djwP2 zX)?CT3SOe96*V&rfG?m64Fk_Vieul~LA=?N)(;Su7XBmQ>yh29qDnE1GUb9stqoJ6 zxYVUH8B5&~)|J$(**;#&*&_N;%iT~c4}BTA2$u|MK0481Cl?r;OSH$E2rXKZ$WH8O z0C)*&%8b&hQUi(K${)@gn#%MiJuiq3$WBx%$Et%%PR)Oe1=?l{Ww(^!;NMyz_z!_T z%X%acCFjtn-i{GYEbXL{u#vzAHzqv#c=y8&wfOIQs?9i{(=@Y(IZ0|Hk8amvhJn3M zSgbFp(K_aP8ihO2u!tJ^7;5|GrD&y7%}Zf3KsCB}LNxao$JJdp1^H~j?`$=hX%w>w zp`0epxwspyWbNXQX?V!8{t;?;+DLUH2Mvp)o4C1^Zq zX&Ngnq2x&rYCyAv`J%IfG)hE95rip#y2cLh4$0l9oBje-2|)o#W7BqO}(Ow+-wut-0FkE_kzhLvkOlUQ&F6(5wVbh^Tb^Z?&|L(-cDlJ z{FVGYer%ToIK|n=295Fc94cnMzgq1=ANquB!cv({aWV|G#3^(UQsIIjq0K|E=R`W1Qw3(^4&lHrU!cUkfVRz;|Y1gRsxg$(*WjP<& z+iQ`rR8Quh=D-hpMhnW+y@7|>g090kXwAD-|BqmwzZW9HP`JH?xX*8WDXo#BgG=_c z_O@x@b~;rp_r~Rf3`%Phi6rV^+v!R9wtZ^@qsXubnR<|u$@ndmT(#5IVOe%_aPsG zM9c#d3Yu>yqlw4G+aUWolCiI}TSNGzO1Ng;2MDXl0W~v0Ut}ZBpBP_Nf~3Kcitsq$ zVnY>j&PA&_sIs$VSnU4OW;G~z*M=jK))2m0HiJ0WULd#l=%BA4MV!s2Q}9B)Z1+N;ES4j}2i>fUSN&7V^@iPK4d z0av!fOcEoL?o9Fpf{})b)xvb~;4*+*f>NbOz)m1;>#j|FW%%Q;=H7+q{=$t5tKh2a+s}~Mdq%Nt zjIR6apn73_5y5|wycCC^duA(80s`~L!ryrYn@CzfzzDde@+*_!zX(H=7?MWjr~B@9>b&A z5#k1E1zi{+S)?yyqOyzcUHwD4KP&hvPjr;s9i%W$N8zKs@1Ln7IsA_iOeFWq*41Gart zzre`n7nj2iZ9#zW4f!Lzs@a;3pH2LL-Xa9f0eO~^KXg9lV98k8DmKjBruJ*H_$Q0MXaL}Pl7KVfA}okNeL&hyAAxX}-qUZ{Agwx*8L*{g;Gnt)KZ2D; zzITIOD`oz%V)|q%Yl{Mo$uWJwQ=PRtO0vm|p_wd_S0~;1Vj%uMU#9ZvxJc82a}?Scz6CM&otB3Bx3gVf%Tv*=mUjx@!r~tI*Bx70;C~0 ztELz3zF>M9HNSd$nezEwfV2ogv>{35`+SXMgxkTFE{y~YA1KG5_iN+t)=a+~D zKWa#O7H&F01vh3F;c9B)tO^K|`O6~x)5<*m&VKOR@_;K2T0Ym$~XKWW#S>!1e8m}g4my4*l8V!O5Jjv#@Kmv35Xmjqn;-)0!_~ky* z1j1rET^q*|%^v8vB0Six$9>S!ylh>~ABOVZ28avT#eeCxfs#ZL+}8vsG2jABsHuSK z{hg#AG?@qUCH>*Us_}prs^xlI%plrK{o9(lJl_d%EW25uyus~O+2w!=^q#uIY7E=G zyGh{rNPKGUu;OsZoag-H0n;)@&9)JCfCd(Ss}Sk3Ah+UL?%z+J*)`!gaWVEHQ4QQW z{1_QyUNR6ipEEYF_!I!#svZBL8t0m-V@ah#2B5(Vl(=c2MdVAdmtoj%$T*n?Un?Rw z#lH85fpN~ijQq?L9(ZjG3?VkA1qH-5PMgHgDB##YkO;S@Ya#bMVq#Y(9Rp7{q1UpK z%5kCrEvH!i2)p#R5)t#T>m(&SLYvBo=>Xk;o6a25ifdl3{}92$IUsLG2ozu((n3A= z1BBPa!J8RU9*Lzvr#BAq?PE6s?#6rL_<)Hj0MDMXbfN4sd?+JUT&f73zH&Q(jo5pD zFD8ql*f@SwygP?P%>Ly|CG$*CAFQF*_^+`z&T?yyi@BHT^0d2I$n2w$ExQLG&!a&0 z=OvQ2@%@Xh$$`4ORZXH>?%`U0(y1)}H0Gh0fHZLUwmh*d3jd@1hlzX)sMzl=>ABks zUJ6Sk{Tu)>2-0gPa~mCo2ka^3O#FmFq;!sYbWKhg#KtJlsQ!zblm&@M^B=1nQHT=a zI3sj-{+|)8v-P;2I}li>5ViO6q}3EjdBqu{8zW2aTyxtONtBpbE?|YlYh!tiVvYgy zrQOrW(wrrH8=)PemZN}FYz;R~ltaZZ$C$#N_#RO_;^X%{z|Ir#eU-SALYw;vn{}wT zO+qCZOL8E_BJYZ#ZKN!y?*@u+)EIpeOA$x?G>B5Hp3r{*9>re!*sWHmV3xoV*$BhR z%Ndfg4&BZWsoEAsPsQVSZN%oV*h3$zKoqK!vb>wnLcg6vH(MQ4d+WD3LeghJ0+;1m z-E1Th)+;r!X2h3;D`3EdeQ`^CIqDj&@-%?F&#??^4IB zOB2@83!7KY7{E;gN-g7>sE(%Jsumw%#(WM_s!n7Bo;poeD-vL~0C^pI zK3Gjo!E9UY#IDIS$%|Y=0Qx?xVJMwG3X@MLohKt!yb_r2b(3Hqt6`9JMpLyNldQny^+9(GfEJVlaEtK95b?wF z!PU4ceOG?k1l2-DeDLrBLqVfeoJppvtd1MC!lLdh+se=1!*nUxPolTScqUN@QFc z-*xgbxpz5QtcU?gx{(pxG$=QTV7Am%J{n(Rp(M(R5abzDLq!J(X2A=-?t7}bdB;0& z)Z?)$v1wHxPDzUnie(g2o||o5H2M~soN6NAW{9}uHllE$>;~&Hsqq?ffS}dDjg#cD z;KDP>u(tQT=+`+7+aS{DzCGxTTwf)`@LYWBA}nI(;ZrpqKl`QgNu$sYdr7MtCQM8b zb&(uUtAhm7PjI&r2>P%JQAnwWP5GcmWJ1k?aWxsoGiD+gbL~v*3ra+?ak8w~bPQA2 z9z`OHXIzAz43M*P(KwdCEw;sEri~Dh#${oy`~)azBb=QHA{8#~K((`Z0wQSvXyM$w zfZFU^Fi0r~SmD8}57Kgt{51VVv$k}kT-dcBiF<#fQ?b0)+=&0!e_6%1oo~&(B-=a- zNL$e8J8rd|He~Z*i{%y>5~hW&_jL67z>tw*W-+*E=2Tf{uS&qRJqzHGvkw^I{Qy_D za_mxVdlmP7%|_T2=KW<_;#-6~*`8-v-Vws#s(!;#`RR$I#CegXuG8s1y4sA@dU|?B{CE1Z@SD9b(ZUzN zapiYv|HiRlWF-I!{L-q`z!^q?f{jz}Mg_MuN!M7Yz#`E+xB}8~rK<0lvB}$;Ti- zO~lrK5b!}PmDcawCTWW29MWRi$S5U&@IQ8tce)K^tNm|I@IUQcwua|ucfMHgNZu3S zH-RkKJUD>~7W^_22_A;)w!L(m=wCPzM)BHK^5*5ePYq{{N8hox!T6sZJJxRurFxIV zbY6tThq~XlMk^U2CAzNO&C0d4bVnpt*T$7AkgPccWVC4LDPV{BK0eCCs;Xhth}R`?pqBN*7#OIxm|SzRj~rDs#2=W zk3y^q*og@s$tif8XHeB_yjhbN-}bMO1|-(dNABljEHThSrh&#FKGI(B`|GtA9l&=M zqL&>bm1MT#iN21|rWSEMIT|AVq830!?{`;s0 z(E}(~z&`smarmsD9{;6-QrI+m8c*NFlq})`0=pf#3kqU0t&U{wXfLXX-zcsJI(jK&&yuNyk zuA~_UlV;E>iN_l{ufjh+R|&U&aR)W)sVxflPb1CGG=PRZ*B3ZKEj3#{a8>y2;ogM{ZTO*qO`T zjv6UdI<+XntZV>=TZ3CG0MgRvz zVz%H=!t4s;ikB;V{@5`mf5jtR>X!lEH`5VaD{3ytw2 zQK#Z1;d>(T!yHq5k5VtroD$Q~WAZ4tB4j0=g5fcsLN31}Z)xYplhAvQ#Ky5ZN$#2t ztw^I{kc8kPmwSPQ1>9K1qKxfKfR5{}>TxlbZIFVm0V?8a}F zIQ%${ZtpBXvbCi?^ms52#1@T*^jpI`>xn^j>wvZ_{eCnQ4ReGz}aEUxaz7B7SJzA&1P zz7y&xKGG~+1O}#A0DH_6{W9pa=?l*Fr5^#l>?t(?Nqe~Dlc7;dEg-FaTl5xu`q&w= z9;AS$2Dn3}&qZ>0^|G=@>XB&3mt07(Hc z>pD`-ObQ>YpvtR+fm;*vM>XmPAh(d!bBI8QY(TzYe*o0;%5!Nd4ENKfK10BzZLM z?j#RzzH7Wv@VcIm1f106hH4Pi;`u4u82%cvCj=Q*9Vq_;{>VNoP^R6&19?v(rWh2% zl!OWm;eA*mOj&?V>ES+7oHcmVo;%~2=D+PgJN4?^(wL$yXAJhGqD|;uD>}T}X}(p0 z&5yT_#(p)imU@+(#OzGk|L`JWoy zLPv=txt{98FmSz218dn4m_0c9?Ff@kOtMtz_%|{Cs6&3F%fz-*^6jI}rGr4wF6CQ1%Bnau}`) zL9sOe7!nfPqCV^X$98Qe(~L%~{I-Di;V!Q3x_PL`#67Q1FCcAiDs%!KCbArDF%4f_ z8o@wy9Q26f&QPaX{rJ5}Q;sdIi*CMSjd{RF3X3sa*#b^9v4I5*B3_Q>*nlFC@|Ohr z7eN}+tX3RGc9bOSKt{Ya{qJ%Czm7fdnf;~R?Dip}y|>#_9s3ajXEchoE-c+E7Al4-F4UA_ z?}SvDs#=0mOc5zp7}JbqI2qWnba0j^(IlE$cA#hPa*T|v8De1;oX%-pndVRLI0?e) zP2Y|?`89xj4eAw6_h@-^8D4P%4lKhA6b}gBN#8xrIlB4#gD0|ewu7~+S@0Ds&8o3j#{c?k&D7sV@RAPnNOCVYT^WYOBX`Zwa*%gKweOC_ z&#l`n-8#T~`24x((!8<7t!SX=Y@WXd+tiW0sdFo|Cibl+Og_~GUe2HRcm$Kzy4pZP zw79NEeBMb0Vc0-RA+C9W*6JZ>YS=3Pk>uDZkku=*nGj!M@*(VpcXX;zf@rZ^g}5a< z8w9IHBi-VLD-RDzjuK056Vv_XA-CA?bkI1oUVeC0R`$r4e-d$;PE}hh8wgAPy(h0mc7SFmTN_Su}87#>9Gc=iJ&8 zolZ6U5@2E9u9W}x$SXL6VLStjx}IThw($ScD>?OW?h7afBvwS#&4B;M#<=Bh62I>W zC*6e$UVkAeMQz3#ZZW`Q0aLt?6i%o6liW6>MkqPLjf`h@rG>{!drL0~sR~DHa{4@HUwQo;s+*JB^WY73Zu)q- zcGZ%0NtAHU>zBzxV0fGTtNu*T zx%xbzwz#UcroOcZ=i6KFVZ_{Hm9g*c<(PG^x!g<`7Idt{I0vIE-- zccs(D%(-J1I?bQS{a9@DHZbaXTFNoy#2hu<*BNTP&$KJ^)!SJn12Q{0yK~g3yX239 zA4&dJLA|WNqCWAcSTH=>om{Ogda;ijyEVxPT0DmDolx(NI?Pw-5IEz*F;-V--*DB-Bo7NeyyuoD}HB7AAJoMaTA{zHB_h%nCb-eaU%GyI zEwY%*faDnB{a>?^o*@m=bCgczDRH&KbNd23Z4a!lCKbEyo)v%U#2%6zF5EpUP*{mtSnz;cW>+poj;1JFGH}+TOf;b7`9t{94+Lr=GD4vzA7+(rfm?i^2Lc zHNU~fOpHv{6=zJ&=x~;-GJ09@$w}=hzjxlfnm9glDtqktnc$CsDTOt)F;Un36isWr zbYPdR%Z_L?XZTmO=IF~^tb6}U8T9%gn7hfirg*H@q{+;gcwVsxncjT?|4t!UJJ^+a z-YPJFhPlM0IDOvRBVK);aNqaIM_LQQ2;L<9V%4{@#*5wNX+InyI(J=J%K1CIw^UI| z_mg$M{k(pI-BH``rA}o1ys9-_B4~T_NdB2%&(G;?UcczdR$KWTb#~ZEFYUR$wU-$d zIr2eyphMcz^xSJ2`?%nK*B!M?%?sTle?2qHqO^KX&K*6onu4^tXq}NvA>7eiyyeAO zpWAk<^^$PI2|PEl{Q}8+9E&hAsPp=auxFqDy=Uc} zbYq8b6^FZpvz6fP_5OS6exvDi7s@u8jhJJ@WC{3B#1+GdKf0<{F^5zz*4l+_!Nb$N zSKCJ@rSUUvUi1(SjXt~nZP9La&2rS{-wLVE%~D1gt_xI{PTnha%%Cs8zh`sX?c|Y zK$yL~J^GiaA5J*j2$sEZ)l+Bo#Lw%0Ym>18S0vq4{!5tj@(}81u#=n4@6;>FLn(=k zKMr_(JUgmZioTV(we}}FQ*|=oEGBk^ci+?x=a{po18oTdDO zi5m^h%{CUeRP#@4m2_wsENgD*mj?X{44N>?<MFIs5a~qpjb)-?q5@?l#xZMJx)*cyWeUrCD%2WbEkd zA3?o)a6Mzf&~fuz11~uRzY_}Bzc{Nop1;1xrti}BJ3Q&5m1H^;Xc*#f);9aj($@PX zbstR~^Hr1YxBHyg)p>DUv5S@55ri{qo1BLe8D9JSqmCS5D9@8hhLvR!UV)ZU{p-$~W}2>PAS<%bjadn#uO z6KX4~?mr0d@YA(rg{0Qo*O_tp>M7#tZQc~I%jpddR&t)@qjM(pJ3eA4Ig^Sm703@C zaWBZI)|%&^hl>tf04>siatt$Kta8@(f14vNPYdrYqri4(J@1xp+@)eV)m%Do;*ckY zfPLkDHn0DtJN`8cpEHK}HBCP8P}r=tCT_8eIb7g@@80kEL>XK8K)J<3a5#+K7g`ym zRqGq~G1QQ*kh{*35Mchb7Hj%JAnq*4?ZBm6JYq z|BQiyHGdSHu()ntz(2RbuAqvvjvthD)6t3U(x51B1il9QP`vB=yTMNqYER~OWn7;h)c5v2Ii3OARwIDwpUsrUDQyRa{n%SfSm4RrU#N|SJ}m}& z>2m3vvtdg}J(XkE^IZZzjhQg&-*>}mnvLZSvn=Ja+<6dmMe~Y{owJ;+9{2=gjW8N$ zkFWl2kAYVf1aQ&dgQ3vPj+q}@TT0#)&c4-N_;H64u59Dk(b&4ZJBst&?iJ@LA}G%f z4JK^M)b4nLaY^<~YUU>{2M#~+|La)pL8B6RHEV6>!fYP-#Z}f@1kH%sVw(Qs{-(~A zNiQo*e8uUs?#A#tp7XS3oL+N&L$`T%^^wGog&1aDEO}r=x0P>G#E|Mb{C}%5hNt%L zUs@w82k*$B{4S+)ZaFSdH;>W7c|ml%5G@SxuP%?o8g1C&ebTvYcS9#j6`o}MtY!tj z$~x4)&6`^~uM~d7X#*xQuYT$vzFKhd`Hs+@r`iYw@N969cC^c={i0j8;L4?xl(OEa zYDHrO>Ui8{US(*#bJ%=CRcu4C#Wvao@p=2OiJat&g}|c8i{oP#T*!ppDq~$+4&fHdNeI;b_-x$V^yKPDacVH3|3o*5Ghl{Rbj$M1`)==!E21Cj_TyN~`HECu z9nD_SLCG+WiFq7X4B1mKWr|!T(n;5hWh?_=o@S9t$94pDfz8aUm6{a5r_GOWhu8Jz-XH2!OuKki3&A{=lLo*YQJ4;TibZU;N|HPn3}Tp71foR zx!1*WLuXF6Y|T)2A0S0#y+Bv4_?~1VEK&G$mDk&~X)g!Ej7X`h**~$nkE?XaF6vsU zB%XOz)1)Z|L;ia0GGtPXo`I0{^l(YpMuR6lBl@+_Y3&QSYPSKV0UUlZS&wY z_uKFWQq66q-`C!A@y+K`h#4Q

Fwnvj3I*j0@J8973i_&maLR$oGy?2~CKh5acxmj( z{_c*xO8QUhfBRX+s?8i+u;%>x817CUYt#5P`Tdn?akhbRUQ6>CjaIZ`DXD#=qbb#+ zxOgeA{25-NE&}mspuZQ9d)2=xK^ayQQ&#c57c@;gtme>O!NDrnLm9~GPhz#eL}!xs z!ECJ+o^7PLTS-oRrN~gDH^;YFq2=Ra38$#)9aj`Rihild?u6^%*-lvYrD~kt)BJop zdwW{6A6(xSGh(lg8F7>h@zJ?bmZ`O#HYmk?|0=~0(!7{s*`zn9SY>hW!HT2yb-`^N5q8K%F@0wA7d7mcr&04OtW~S)@pS|=q`P8+1 z>Asd5L1?4EY1zv4eh@oAd{*|}cwK{4KO$oIem4#xcz>U)$>8F!7?YqMRnFe6 zzwEkk-$ahpae$W2AKkD+SxU<5$1CCkK|M-RS_i##Jld7BmNrUx9V}TI;YDn}k$liP z!{ZR#l6FRe-V%=dfQD=ILPvh?Wgwbsuf3DTo`fu3A|X*~KHJ1^EAeqCwR0DDTS7E* z_YjNmUb(gqa%kVRf$iGGn$d`5CnR;%f(zYlopX2p4sSOL6mcdX5_29b=;gl`bXO>E zg1cMXJvWP`xVs&n&9d)EFD%y_K#(qkhAll2f$@8tGQZ_&?~n;2_l((TBZB>0-A>#D zX?on>Jn)ktef)eHc4Xx8>^57w*5CPE20W~ zU%f%Rq~nLsNmEq?;?pWflwI3+21g@WrDv^!A_#G8lZBeToS%l{t}nlZ^$djDL>H-cRrkg6lYB@D?y(gWZJK@Z4Y_OWvZ!)sdU0^or$ zd1?s#j=0)G!}6Y7cd99myf{jb4Tt4!1Lm_AtiOrc%g8M z<{EXx0_1>trt|O5-LG?W$hInTv422Xdq0`vPHt4k<)O3J#^EcW59${BPRyDFJUeA6 zoBpLQNSY8G5wsb0DMoh6n1A!pxboDt^mdYtO7CPCmvi-?yjq8NQg;p|*bb;AZjNYl z5kFvQRMnS|cIQy%QeEUpbx3Pt?X-j~xTWILswf6}u^@B+r(Slr&nSSM%mhcw zqz7g-*qOTxh*>0+v+DG1wR`_@cCCBZ9`^?}qk&zceO7ZhC07;PQwRQJ1vB#Vl`o93 zATKLDCuOKQfwdCXu>m!;C$BctCGf2Hhjqnk`-v;ms@YA!rF~twN>iSfgs#~v>I-%1 zn^TIuWdRraj-N|eispC1?X2?c@J?V{yXnm24_6%tnqBDZ-5jIA=-0K?Cpzfs6EirV zKg3NK*f08(Isddn`G;TkP^3O@CB9hx8K z(X*VjV;J2Q=PT=o?zJL>vXitP^Xcv?gGM%Tjd$E1LPN<)=Pv9Vh97 z%Q^+=)ix>J)HS(x)cmnejqk=*b7BjGE?F~Ewx-MZxcUdHx>=`80}7uW!u@5PArQb# z`R(RE*2?|!B7(a?_UVE8+!OU(mzv2AWi9#?yGsuPMifRsqncS|aMaxeoyI*|He)(i z(q;pe7n77CXwt9d(wf?}OOp?kNLuYIM5b4%U?j&R=p_;bsUm6HW`o(#0!uv9nd{Nq zZ+SBH_`p!=W@BpUTDPmWpK!L9RB^tURs!N;Z60mS@F)6Oc;rMk3&m`98@@x;8#fDT=93@!>? z*7Guj;#~rIuN19!-tv$9TylBnUAdAa`>icfUKumID?5tWXZ!U;RvBqfW3E<1M@hw$ zgIO^=y5n=&t2t>b(jz2~lahD{vX!A7_9OEH_+oJ$XTdOXajUQeD|6P3b=*2En7mq1 zP$Ouo^hV%<)U?(fT)p5pWK*rKX~2x+sJDyl(vS+hy80f*xK4juu7t~} zJ_wUp$g9FWsc&VO(`RUcc_nxp=z`k++y%jco$v<;UVG*HyS7gfE~wM`>ju{@4e+~J zH>+XbsL4CJ} zXgt>$#%OtfOI`{-==s>U?jv<2#|^5E!#*-$@>5c7qj&ZDgTj~SY?Y!}W{A&Lz+Tty zN`^~C-ilr?6%npZ;5WrNfvGqieC^YCPgKwlnm?bjx)o1)T-fGX*n7%^>a5+O>3l9| znSVOV^;G>9%}0&5v(F1(TP`@ODgijJRIaGHeBP7gd`-=#h3L5&ylPaR<8}({%KRK5 zWY;$0PI|71eAce~Ll55FffZGh8(uMmPNI z*z}#seQWL)>tqAJW1TzYl!dy3dhdYhqbY?rw{JAJvl6@y=)BLbaT-O z?Nm-qfays)<7GlIrTPA9=-RmKQ@gDL%r)s#uJQ~f@}r!7lLRFH&#rpsv^HBwLx@M6 z%)FVH9Nl$Kh;?~~O3X^1NeW`%2)>%P{8XLVCpU}JG%gwG7-HziSy*e=Xm}c{>t$K{ z`K$Nlaf>=Hkdu$uCiZ`wDV35&>kKninUgD{MA`f9k=tq4GaDAnjjJ8>&JQU{{)Gx7#mMCi;k=#6dn4Qt z1C*Kk@Co0F8DYk7&%rA@yhC9KPfGyLm{|c>G@y!q{w+|y%8RSLNdO2C7aQR0E)UV# zW(voe-P#7(@{=iC)`P_zIN}pib*v+UI_bfZy%4ml`$(}aE%X`6%L=7f?@^?gI==BN z16#X4t6{mOBOUkagQ3i<2Q149de4r6yy_b3jLLeTRm^YrN9A6!W`ArOUOOD;E~S%r z%oa#Au&~T`uDV0!MktYi_^L(sxcFtGVd;~`Un6LV_wyTfb{M6A{`Q_0S1Sl|AIS54 ztu0dPgUmnhP=v@VGxR%>xrXA%pfjS#zWcO!>KmY&8D@2m@JeqoLs&2II^;qRc?E9g z)hd$aLF6ksCt;3d8hy7I(6yzJ;&k$1ljQ&#_s=O>kmFp6h`w&ifG~=yVd0CSr{2%U zmHkrEL(Qua>JZGw9lkd5n*4~Z0HXZyH!AyO*vX8Wm*UNBipa2$9dp+fDA3qj@zp^! z=7gkXRP)>g^g4E<(@8KZJdACV(Qfh{+-i6VZD*C6u{h0Kw{fU-_JQRk#u30<^8$LQ zS>S{HxARLz-LLWHpLQmmp z4YL=~(ci(`vb20#Yl=lTDmp((x4T13mZe`cRcn<%s~huIr;O0#R=U4SG#q^zSL;C# zJtz_m(i|@%9>)XD=8F18XO>kAjG)0WhZYnM0jywjc;wm%fP!GZ@ z%yuBz>C6ChuUpfnBMy`?N!w^Du2KMt?ewq3UiPpb@RjRfu_}B^h~s~C5$t)P8N5K{ z26m1J!u)cjHarVWgjB^si|FM2{3pQQUiaV z-PGAfco1a;8IYka-0Mt9UJ`e`shxwoJD(LnO6@?PR3B*W*U0P9@~hH->@TdRjxIyM z#|Zc@KL%?luGm+fn8}Iq*oYm_Al=M&G73;eH1R zD9$$&-5#hS$3~Dcm}7@kFzh|zeS*IFj+J=Q%-g{*%Z2fyq)L;qH$X59%2R`#q^js6 z9ER@ITh3-G4LmNb3hqk|V#}jk9ecX>5RhE-NcM1BeAfAZ*xO-9vue-xUgL;&nIxYc zl_N@pvdf(Y8FHv=4Gq@X!RJ)_kGhgdi*3P~cg}KNprZ!REYIYVs*35FxmP;oIk`2KhH<}}{^~FBC$;s1S$u=V#>7JGfOhZ%oB#5hu@9^_ z$91Eyr(<&IKQ_&BQYsrn0F6}nh94sEPdt$qzt>6p6k^ts2*+iz%bK=8G*s_d0Xmsr}K ziQ^X_O!yZ@ z@P(=flI7v$w*l_f`Y&*6R{#WI5#2MS?UTHtt16CcrKMM|3;hLi^ya@_UT%DyPXa zu4z?nXCdVe%zBpI4HsISX12>mo+kO67DU-&fNe`I!#8k>h8Y2Zun*`s|WHv0Id7WolRk(M!Ldic13 z4^A8QMqbrkXl;Fi4YDNDuW!u&l-=s2bF1!-;-j|iE8Bck&iWN)#MzH;YOsdH7dAc0 zNHuHrB6I>AcH^68By}|0zdXc@LuECKSIsRx5P?q30(wRk3;@k1a#@$AIXgF`zRNj~ zI?k*Q@lQ_E-b=r<+j3O7!J@CqQbQGUy2E^u>@Ww=u~tK&fiFbyhzTQM%o{~{`SA0W zQHVloHggm#4Z-}7k8@fBP=jmt{Ff{#uqZ~{dOJG&;;x~&3+`p4=D!+}&lsunH1A(7 zI<~de_O1R_l{-tTHbD>0ZBlYI@w(`B>wu-4ns;)4Qu7Op9975U$U%b2q0l|KozEZH znyM!CLTX|Tw}cVx&k9RIk85Kq8cejr?_&aEWrdBrkZ6^G521;?_`dryIeoY6?ISTQ zF%6CR4gN(JNoYQ`4ZHj876U6%=;ueuVEs<~y?%EUBg1>}Wv>Z7N$v{NiuB{75W`Xf zB7Sq8zlVmvD!`P2C(YI58+5tw>aJR0k2Udy!3;8J*rEy)eXY|j}{k6a9%l|8sMLr_RU^}hg$ zZT>Q4!aGk9BS{pP=!_|bdw)dg&9Ns71SrMaFaV|@)eU7`d1x0I(3tCk2|~L5#Yh075lSna4x!?`pT(=+`;j1ALPZp zRtvNGA)r)fWGvT+k4)p?VWF)LP2MjQ=Yx30N^5EakaBm zUxwE!5H5E!M`+FQRvDxE1oPQJ)x8bpda;rHAN#?iOt1P%W_}Mu@3`7~K4%aG{#Jyf4qZ>O(|cG^#scAfZ0*EQPtjy& zP(jvmMA4D|`1|}^pzIDzV-Kmq_q|zR!cdfG@70N)n@-Me+{gH5Yy3Yza4a#DEUliVeMn}+_3&l>aTdgR?6KUaNB z^Fu$$byU+g6CtcBuLw0J9$`>avOfE4E3o^h>{%Ndc_#Er2&(y$JQS17)--I_woN^D z{*v%jErrxuF1|AIVv?B%Qcy>~1!bgeBzzoeWTBoH!Wvu%7@;7Uba6+3f0+jKK=``5 zVgDDm<=^H4-pTp+Eb3zR$3Qpd4Bf67lM(ep+hT%h_<++b$N9J^=xpD7>;0XaF$d4) zd=8?jUIjlAj6wFMr*aR;}U)xg0_sr4KGXZyZ**EvvI2&23x44^JZyy4gmm&&}us!3B) zUVHx$n51nEbBxU{Bk*qiFXrCDE$VLVA4O42ECeaRqNJolLM0St=x*r-Ny)(kX@OyA zP-1A1hM^RJksRq5N^po70qKU{B7F9<-~H@!-g8~&T-W&zX4ZGDd)@b^R(kn)!rQI< zYmZ-={3zuq;X{vlEu2xZ@_Hy9KHOV)ox5j_i0JtdE=^fRk+9Vh*$@G%DHIg2<0i~- zv5c6J;Zh(#uVAGLM?MsyNo8r2t|PJDnz0U?&aKR+Gt^fljbOjU*cRjpjiv(q$~UGc zg|tTQ=PAjphRpn`$QSxGwimn$3vA$8%A!oQg&)I>MYd8~CqBOpoE>0u7jmi;%bru3 z;;na!V;IHqD8X8WC55_oW$E9|Ch7z@MFsx*MtC)6@cV5qs=077R`Z(pgsW|huQr|) z*1IuXxW^Dc^nbA%#P3cc#l&A7YZJH8B&gWlQLG_Gu;4Cdn`m3=~b-O)TEwiQ- z1NSVo>#(+3^PoeR)?hUkl9tLAbC-{KT~xLHWa1X2(aqAqQ)d3uH5D|YA>t~;t?caW zbmI@SV(Hyy2t)ZJT%mONY{i6eTIx4pU70@XUMa@v`n{}ioEtlGCr1HtW0MF6>YtcG zpd5S`rAa@`#Lb`SW9raOsxAO)o{=MZb?gYcCW=0t<6|K&UQn|g*SoZwpJP69EKzIN zqxbr!<27t<)%m#7-SMw%CV8ot$w~-rm!-%U)~c)MYLlc& zNHr&9G{w@1pm&b2`XIhE^S!0!@u4Uvi7XRikf#LptLa=()KdF39xlA^mmgY`J<&r5 z9-c*;_xJNzk~QX|#p^Pdqkfx_rCcY2xV32A9eB1e7&_V2Zk;Z?a)uc9-6kS=_cE7K zV~7@(*0U=41+2`B-@EE#YhE0N71NY25B+(=sI7#{V{iBE@^*0AZ9KJDdXea2{8z3R zVry$HvEgrw%?I7Cv@E}py1oYzlBDTqln%y>hbN9Cp+wLVwuF|N{z5KSTNT@!xTIWg zO_n7VuwLNpi8L^r6&q$b`C#?sf4fcgGCoi@U!fvE8^voUOPrGv#WTzv^o*g$UAd`d zmK`S=G|&4zd@SI*c{8F>ny-vGTKyR1#lLo&Z0-FzFnX;(NyMloa8+d-PG+98w$!xO zK?w%R(<{OSULJF&(w0F#n~~Yb#Wh&;@GMV?Z7keOnzdJzETwF&CcxD|0~*}BV6_9p ztD7rT&Rw_T1Py9j_`QqmGWBXbkk+n~4Qo)rUV_6%pfc0@bEx@6xQbP2$xe#YP=`a~U{vyL%K;MBilp zYb!-h9?s`cumBZMF&l)ZC*IGP^DR3G&`wA#QPIKD4$T&LiR#UUzp~d&pNiN5DCn z`bi%xbc$bbF?_;)$c14i$I7=lNTAQuAVM$dcBP#}P>Vz5w7Wx%+FNrhk3yk)I5>Ss z6Ef3=H1m+@b0rslX*Q**WCu9eW$ZH_=Z^~Zi(jc(_+kY$a@cH94ZLxxax@>%-j4+u z+pV%#w)|JTu;${%OMA4cWS&Kt$RQ8P+|3~7vk%n&tyO{BChKZ1M(`h~sRs{amU|iZ z)KjADuF%eF?WlGy`GTc$`ngU7lm$P~M@ZKf#%2J+N-)au<73RkO-N(bB!lGOp&V^R z=Bl)bGYqRtl>5~p1un?F$c{lHaSC&OugcFrp4o0z7py+YmJ#Op*RRL+Al z&)b?e0`cSVdn$YkQ{y^Fuc<=v26F?e?6F^v>^HtfQbO1AIofwtT^dGr*Dv(uCQ43~3vMlFL@DDb9p4hLrk=`r*K(m?Ve<%o=VwkZ}6nf0N zRN!s4OHv2IY9Cd~><);5uW*F)!YX6_|{Q+y+L+ik0M-X_b~sH-}qHR zBWx=eJD+V;Ay=po*UQ?pG4o%QSaYo|a+N3FS2BY9uyB=>UjdI8s#wMz3Zu zL(82L|Mo$?US|;UDFI}Nsf*H_5@CbA2IYJumqpZ1T_+slvMr9D=7NzJsEK4&qVBA9 zkOSgEO?B#wQcwB?KDc*H_3mJ;O=jk{yo%f}KRKF~QQO>H!GW60PW`&dl2XU9`o?~G zu{+i~;3&wXN(&DR?@A!9wY`y6R_TGcgEue9 ziCZQJL^?zyl@i~;AMw{*ZbI{s4Hc#S;lb|8i>Gw?=VFY2_|TKUjlRJ``s#L@Zzu0G z${WL2i$L)^wDw4=zyBt=b0SHjUNbxcSZ>m5#dS*68a$Zk<`3w1_6K`ec+g;~|`^*)$dn~UnRClfF zcnIy{m(dfR$YC-c)4|U5k?^Gn&$GMgBFL9Ej0Z|o1zuRI2?(OK+K@zYllg}T;F>16 zSiI?5z`H3z(chj%Oq{Bc$Mlx<}y98mKjXf8Dpu6V)uo# zEphG2gbU-cd0hQ#>5FW8;A>r<1)x_jwK+WpA`(!jjG!!DYI6g7c*@9tZ^cykR(GL3l3SpALWt?QrSyb`Kvy(xW2TGyN5oRZlH1R*Q)ObZ zy&`B_&qRg3%5|V|nfgw6Cc!z?TZ4=rQf$e@k>S2AKB^v(G-a?#TCe68Rqz+7rLnqh zH+B35mg^AaGXCW6D5ca6*$geMV#iTT04_slJ`DNL@&YZk5a_2J?{A=_!SgxIa3Q3^ zieMtCqx0?}D3c){9WC4)7#au7rW#!`fe`7s6Fz(r^w^U=1lCpVk*{Eqi@eJr>Lj_< zV%lw*gEF}SY*;V<^xQY@KLw&5FYn5gU%gE&Zdi={xRxDU%Tcn`-Clf+9QPf-$FCpv za?Dz1=1v=y^r#J5m2|6;-5=0Zp|c+?9^!`j4oyqD`md3xDqF{yC zA?yxjVf>egh>joNn<)clfV*P&E^AtzwQS~;Ut0HGJumu6y{}9(!_CvLXcubz>|5`6 zSW|$nLI|uCwtWohUd#Br{1%S~*A&lq2dT-q1MQ|yvJ$DbzU&)qS2k(R9{8kbD&&p9 z1a%4mk}kv^L%W?^8p=^yk&V7%W!yfXnh&$J%P_ry6F>-S>G!>d(m#V)X^Cg-{BZTQ zs?PL6nqJrc`&Y`t5JQ!#11;KB6T3Z994~-@0;(BRvS$V9<^Q)P>r|(trD){-BbmW>mPvOm#cX@TX_Gc;eol&tk#c z$zQr-e0TX#KKe}(w=fLqZZSBgo-+*a^RduMrJ|90@(X0@h1# ziecrk&c^H989!aVM#g3|0ykjxfS;`J=6iq!l#12}w)Q;+sZFyR;y>Y)nbn#$* z3PLggm!B#FVtx*l2bUM+4<_;98s<_kT9pCDAfIuDTJ{EmlBf+BPtEg9y#)=~T_usa zml%z9s1= z$t=&|u8!WX9_UP2@MgGX=swSO|e|)-Uys>J}c}G+>`Vzn@-^lc~@Z&P#23qknQ}i}uJMURrKFuY4-KSZ)#@-t$Ql0?#vuiSNDzWg_ zjR}Gp44wQkwGgB=KO3!FOUI)hp|K-IF#5Oq@@tGgFbd#jG2TWrED%S}oV6AyjpLUn z|420o{qV2my{rL(gvl{4f4Cu1mSwWcQ>tSl<a7sxTMvT*h1;nP|_12vQ6E4g6wrgS}9K^W^&r<-3wo z%Y#wpn_`*Kx5{=kTHZ8B&a7(vh#rM0hCQbMm8ixO@7 zyNn8HU*PiZC5AI#B-X zYp=Bu4_Ovb;xyg%%CVq)<+t~gnpe%6Rt?&Al(=U!T^0t4{>K_-1=embu_6;iyF7e7 zTRadnCdPL_kk+;?8sn4=xn7jI*~*l-G~T=99zky4r1wS3hdqm!D2Vs~y_F`rdg9@C zO*M5{pnI1l7EhF4Mwk(So%%|-OAI2yip%a2FD&Ks3*)Xknyo&>P4xf7p)(kjLgjj= z(~_{WZV^=MVO5R`I`Ry|LaFDfNww};i9~uWFGVY^g-Y2a-4{1i?XFownT@3P&Yr4u zfmDeJRAJ7Q*f5Ggh~8X2f?yhR-;QoCk*he_X=%lK%I-d6HLs)Mhic~o5{KJ$^w*MQ zOJ@_WO%N^rdD)_mWHc3T%Bl1Ahqieu)u+Nxg}h}W9;JK9m5yC-AwrH-o2DHo6)G!gbwvrwoP}|uk;xC6vND*tG-8DAUNr(&5y98d}zX^)%f+uC-+V+HQ9;8pr-$y_^1E1)>|2Ow+GGLN!H2t(cP@^ zqTRBn^48{L83SM_*mxyKOY~<=+YZ+B2_>_ow6=GCYItHnz5$XFGgfZ7U|I}9(Yhp-I0~IAfb@SlBnBw9+t>4QvF7pCwIlq) z12sS=1GMh*1Gu>)Coxfxy<1s&K%bFe!$#vRKSD?4RH{`N#I>*78B#aa<&0ytze{VI z%qD2uF9{SG65gL@4d7jBqI^@RtgI{rf4jdCT0pM->~NN#P6kIkydzv0Uy}ykOrn^i zqhto#kUM3KBOT1-_UJWu`@|W;`iblBjvtoJyFoqEX>p5Cv?G)~;&e%l1_!K#Ya&uY zxC5)qM~!Uh&q(?ziE2+vNz|z37w8tL+PpDPW4hg&a9Z4!VJ+Q*gD5%Zs87PhI$?H} zbkgNet}ZjF5np{}9y{tZuB$c9CAA!0JWq(nzcEymp=ob_tQIMIC!_L%-&5;XnAz8` zBIN*-WRUs$pD_&c72_g?a9Vyyp&*i)dDy5^v#BNAz8r5S7^Gfw|#oR2_ItE5B>z#h^ z;q##hE8DL zG-Rv`8&(@l2Z5Rq5{*wFjyil0kA`(MJkSwnujrd#&NodxYZu}^g-Gl>f(EPQOU@tZ z%?ivv{ET7VM@uswMKkxT=9}HNmYHJ$x3OJL>}sw!+3=}2`lGuna@`jS%gqSA&6af>)mO2?I8R5fzVQz=P$Om_)C>S5pgr$fr(9^2 zeLsA<-nTj0$Os5*ilj!o8{A7Rfh97wHH@~46poB$!$7?4wRzpAs}|$8^%EYdH2y!a zKcA-k0}%kEm-eXNW(Mt^xVG}amAiwHzwPn$>975@;!Y7T12`W$f)3etuJ#7q(u3%u zI`uHNO4AO>ZP}{QBI_RW)j1-Lz9Uc6hMcncc(AuJ1#y zxp0!UAB~5+7e`3N&HS(IOwzd+XK!Kr3ZrnDcA3V7E8;FrR0u3=r%Hb%1GlYvpJ?Dn zr%qOvy9ADpTS~;SNUfdbf>FmRd#t7`!XR?hSnMi6vs0Y{HgmBCVQ0p)U;2c9Lh(;6rlu&DO_n zo3;wc*D~6jOP6zt!`f!6_3cZ;y!6|3qM6;LW%WnmfY=Zl2f`rGu)S~LP0O9J1JC_A zi2ur*0SEH0DwgtrE8~rDRAE4-lmDgKDav^sI^y8Kq0KLI=Oo-Met`ioy;R}$AkXz8 zMbI6}rLK>T^_lJCR_KW>qz~ur+_*SJsKHA( ztq>wsW*k@qph?6fbg+|uE{&&knrhE_`vC2Z@&UCkMV~~LVqYQ_W9$n=DJB(E=hFP2 zNkWC}ob}pEQ@lK$&b55P4ndKv+T94TuP2Q`2Y_t)vTy?o!WNOvE3L>t|a=)!uIa0l$2D( zI6D#18;ZlZn(M6NHJSyAYf4jf$MNODNy2%t(CZrV$o7*e@!mL|45tkne3f0^(ugqf zb&Iy?uR^b&lHm%sEVUXp=biWB!FWHB>|jo^ou?(%8_;a}yl0yq>pbkQ8?U{d=G=-t zsV>e|xcM>|;j#0+451CMk;MBD%iq_!xXv*EME}zA=6NiXYE)pSX?^5YPB(??5kvA_ zP9Ii3$ti9o=pVNMC67{)Gxu+G(szjsxgJVerAUyuBW7PX`G_-SQNB^K=}v@rpp%mX zdz2kn8cYx6yxl!pmQbOaOjyWb?I=0+p7ap#z4viQcw%U;26<;~6Bzr~)Xz<%!mYBZ zeDZjKEadcJwjK3PjS<(t%*Hw;u7b8L&3&)XeRERGolNSRYAOtYK(V_c*b6MJ14RbS zBHdKHTHN_R`k&9Qp;_!Z+dgvCmo!x=?kon6W=ZY zNwy-yR_f3*^H+$-C9{4(EoJ4!Dt$ni}!ba!>9d* zvC}_QbR@-d(nmbz{OE;uo?-dAUAm5E)0>@^U|5s5ym>RMoSs5^R#jbc=YIO-9Yf&` z>)5dUi!t_IgF%AWrBM~59b&x|mWA?Dbx)3NMi{BF&AQPB*O!>FiFLOUd9gC9J1V3O zu=cNL{Dvn^5S+N1nDnHGhN&LR>?sb;&>q5sw|-w9Vl7Z4$sA+U^WwJH z<-&xXrEf#d-PQ|Pacd4*T_sC3l6^V12(oDxh>`;i`wp^p+oHWuaXq!z`{ph(bDSEH zP^%xf>H)>3Nf{wl9I+EeDnM(57$5cAu&C~ndtGYyouAo?t1^hpIOT)xT~EnY;dXx^ z^nwumzTHmDsh<+`TPPtwN&JeeVmB^q9oC3{9CuBDV#;|jyzvReR9f6!zdR>q4TpF& z*Nh+yXq7CG@l61CyTM``XV)*M8hN#oqg3UlR_&E>2?}m{jUxhh$96#bwy*kS1)dU@XRV z%9uwSUs*g;+~&i^xp~r%(r&-lG}u+Z_V+D)x58@|z)`Sg3@+;WbMa(m%^lP|7Xvf5 zII`)f`M+lxR6ih|uaMQR-mJyo2@`y- z53z0VU)_|&O8UnzKm7j3#{Rv-JZ?E$@Wh)Fi9)K>#!{#M;#Cl@e7n`eB0T2ukt&>X z$S5xu`twHd))z}yYc+^<|TzFgImLY}9X!R@knon=X=W`hb1>&ZAIy*?{KtEwI zUTjYSGCan&%-ASGJ#PdZTxC9BrT;RZy&V5vc*8v(pQJf)z32n)X<_6;L!fm3;0%3} z)uAjvvCk0mF_v9d8NWJMyv~h!zIsxk2!f(v^eZ!kA}aO;=hFO&0aOf++xS@=Be-Ba z`A!)@2~m!^F=X_YRYHtGhRgrX9UfZu@9ywAq9a|=t9N}i5q5RTBR?K37i4nUeq~!r zzZ!(N5qCt4q!nrzRahiOzK?oaY941}o9t$<;1TS3cA-WeTQBg;L_A@0qiA)pyPbPf zont_lX56UbVqaU@h%YJc?>QQEabA_GbS&QQScl_G!<=8cE47Ht%2&?BlsQj3A}EVufq{e0o(xM*@2tN&`-9S@g53*w6acIGYcn0E}APpA>wZ6^0& zr`veYf7tNue~UlHw&QXK-ajC@2AE~pQ=JC*Z6LWFoP1EmP?cu+j2TIJCSl*37_+Mb`38k$Wp;*~0r_r*#<=0Aq{n>cIOv2!6QgG0y}YfU zbm+bhbW`=xAfllZ{fqF+$3$i%p@#>vp_6SXZakyndYd)=&^-?9^D5TY0aCyICl~DW zlZg0x-(!MY&VbmR^-r6&IXkwz+5uL_P9Fv#NKdHpTRB{I(T#jOoNd&-R&Pe_HhYoR zZTt3u2EO(3JpL2uYbFD?aFmZ5y9)=q>-k7zdljPXt{kVy@WPG3-E;vx?E-F!$lSMs>M$ad6AZMN1_kSKuzb^7Do6c|}9g!}5%pi8U+h?~vap76Af0D4wj?X&ziXH3)%1$4e z5O_90SFZ2U$%T@Fm*M7#UKXIT<3cSzZIE?PlRepE!H;g~;IWIBT=F%kcVJ^w0@vgK z*_6BW4<5#hz_EB`^#1JQ@vw_;f)9hg zz&idVLBGn|$e7fliP)nCiDHZoEie6C6CsSdvV&GXRHg{nKFwjp#&bwXjv{veXY6Lp zcM&^re(=l_42AI$JeN~IkNR8>mRNl-H~`FCdH9xojpt9H9+WiNrfA|_G+PRXn%LE% zpHwsj38Tg(qpgEkT1jMW!U3yteY&062JXPJ?`W0B5-I@F4US(pDgxa8M)ABcBg)}| zCr?wUj|R6)eet^&w;#x`pEo0YTdnj-^pF13{=QF%Byk~uwX&!D+&ozmyne9Y^Fox& zH=qSM&h-ccVn?y$y$8LHz)c%R?^`EMfJj}HLtM|tTf9dQW6MEh(PKD%g zG|9j+bE~gJLI0DKWZwNmNB$t~UTo4!WdyKb?S?T#Db2ouWZ3;+P_6x~?AhD2uW?6b zp_@dyDe1*;{Z9`%C8t_D+8;G85@G(e^A(R&1U1lS0{(cAU+vI@{_4+iHIfN!CEd%3 z39^~#3>=5wFnNdZ%T!>3+GJdvIriS+U?i2M$}E%#OjlluJ?q&k5fEq^@cLaOcFb>H z5i_h75Z3uj)Uap#7CtLPXy@L&8yvzfo{4)ozw@BWK(;q)b;N~rq7w6T5Z^QR+!DvV zy~mSDi+4h%bGcu=+}~+q9-(sZuDwe>ruf=-Xt7I-|9NB;zwf(5NL!G)qncuAYVFc2 ztuG&?$OVYpPosUbPJyfA%YRPnIDum%RmKJpYlF_5D zz7-EF2)F_!BPk!w*zPAS$VEu3b&JA&l$f>N|7@_|+g=#2EF=p&WLtLl|DE=9Q*n#eX51+kk4X?yKtf225 zeMk&m<$Koowfx%Ea{9s8%zVSw)hE2B>fSAXtz#(_f^Xd%>0{;SeM^QRn`P&xaRtNv zhq38dt*bx>Q(0`k>Gj_>QAWY0QTN5F$Bo&%gMP-=`{9*fM*|SFs z%=^1k@wT06_mq5TvM)0IDw<#jqa&g^v}Rjm1r)lieokX3J!3aEw)G|PW#{KHCN!7; zLI)uRMVGtXf(p=Fidi%fpe`?^r6{{^_+gk;(8MY!cX}qMU zm@8^dsi1W1i8{xpeVouWlI}vXAZe#4B6u{KZS`GsLI=7bzsc|p;!z~X z8Fq_^iuh15lM|3dA(z%35Yt^bBStOas=0kq*sDG+q^l{tOu3I#L#VHq&7{SXlO?&m z(6U>{Q-G9igfs81SG`W$^iQYz83g1>iAmjw6%w`h{ItYi@#JItg`0*SUQb(y9w60* z-)TT33k%>`RmKZ0zVZLhnyku;P}nsy*4mxB4ysD2h9I)8qp>cG3+56?MG8+2y6z;z zaor`e=n3AXz&C`a0Z?|7adQ6)Klxu*mhnq-7;j9wa(6Uo%q{jyMmgYh$@eX26X+NVdp&OD_1*K$^mhqK&$F$*2T_!Q9=0SP zpT^QM)a@6s;jRB18OS-+abC!eY@GIq@ZIrrktnbSSF-mg;zu*w5IE3Lpc9 zA=^Wf_t;TG^pq|7yt#3Aprg?50&1QD_;cPTru?!yJRe}l)Eq|{D)xnn<76if46z+g z$sf9~v85*Z*{8j{eX8Cd@vA~rn*S!9`#WJ)&gW1EU%~~1Y=&`%D!wK=fFzWznB#^n z#I5r3{9NkDW4hEl>nTAqy@`@jPi&Tp#W;t{IG%4BK2y~7zla)T8uSt9NPP6?^sKgh z006x=;zHm**VONSIj`lxcFD#zFP#Djl!Lwx`KhQ$L!`SVK6QL&M`V3*Fo7DldX~fZ zko4vO3#;EIq#NDN*dx^TKKd3`VO`nUx9)d1pRpWJg~>SCZupmO%35XSzANi72H}V{ zf9eQZX7unIn+f0#m5v^Qnv$J0E1Q10VlG=GgHK}RtO`OjbrU~89n!Y3Ik%7@Csi`G zMszZKMq9@7+dEzc!$q=hWA0Kq-!?_7xp(iR&}90|LvC$N-|O#?6}_0}nCD`7dh zsnTd;J6e7UVPsvHnFN5&sjv7M{W(4b%cyRUTPjqPdNA@3qPY0scC0Pp1Dc36^{<6h z1o}Yo9nlWU^guJn69{P>Gb_lz)FJG9hff_fe#Q^^@;2V5bFq^XGo14+oBr%=iJvez zp~8Op{=pT)7@?M0T>NH#^iJ69QFuqP3Gk7WhR^r&Rn}(v_F*?CniCHp0bn!?kl$Ul zWc_g`*z>00SVCXM*4$kH$zg^_9)3^G+=pL;H-5P$tE5X#%6#9T?(i$SJR4)h|7$-h z_ocGee6gF?{Bp#9DJR9w)VJIm@c9^hDPV0KHrd-6$xRdpzI@kRAY=2sPwvyvK>bb)&6DF?+U>hsf5^;bw8~(*V>x3UDHf_Cxpf3czagHD9wZ;=>C*f-J`s!Kmm2v|=mE*QtJWG2C~MUAmP+7#X2tjmCbO4)@8( zYTgL4NKUmFb#7zGU5R887T`>u99%4C0PFK7GV8NuM-W_w^L3Ecsh4N64OFx{b=0_% z9uvJferUzFAU{*^ve1UwC7&bxi?LsWL0n49(=G?Rx6O4IjtkN5p<7=(X7$zNiBFA? z4M<11b)mc0+^TJexo>K$_QSkLiZjHaSLQl_{%u#BweedJwM_vsCA-z@5A42wwH4oZn<$wpsw|VP zno$Qas*ie5F&Hc>nZ0?MdcyGdes^I8AGmw=N|EB~Q01!(f&1s-(5md$@;l2^_%Vj+ z*H1@uw4n(Ad7pYLpj?$JWkTgWo}eziy{(mg@zqlc@}2MAeR5y~&-msT^jUHrdz1D5 z$9sW@$Y!;#eN(rqv5*181`l8VkMX}N8fu41ona6A(V2HiGDLC*csRC&xMrV|LCbEO zKLQ;!Zfc-kv2<5EQYCDLPBj?B6#2c}nOE*>PAm0yt_y~rJGbBLsEb!UW$5MPZoS}k zW@8Rt_+nY&ri&g=ocs<)er9+ar60g zISf+WE<0Yr8x!aTgXp3lfFWL-PX~i(q9$3I%Um8!UUBdnt){UXq_jd$02s3hec#C6 z{DwloW%Gx>+^vGEFAs)&_?1xa8_@qm1c~}-bu@M_0>veE|Gk1-4XZ4FKE56V=jFfu z$P)~^=j~F=5WH}zD?!gX(B1O^(%$-2&@x6tcl%3{hooj>_C0IEV=Fn;!6YN@dfRV> z&I!4Md(i1hC6M!EI>{7NBs6MOrFvj?SS`nf`y`bI&iGIyFjO;rqMzG-*>ll8)tnz> znEcPZ5Hpot>y4Nju8M7E-t2roIjm`c21E9ni-)2iqED8Yp$R?O8VhFo1d4Z;qg8Ic zTegb|&ZL0m#UF3}i-pGYg0^zrvX4XWB`;qx%z>@)js_Ure$xBo3=hX?>cb0pLM^(- z0cV28_B>dK@E+%E)q3J1p0GW@)9tc!3$M$|Rdi!;=liFZ79R}mL?*-+ZhrK{6`mpZ zwP!D7KA#9>787LVmi>qA9P^eGp5?B54FD^@5A)85h|)WD&eeR^*3jzJB_n{>4nE|y z^(jKU8!$lGzgjDKndtbLBe0_}LiIe{KZLyD!5bM@8GCYTal{(Q#}kL+NY&W_R9=F_ z4(FcVI>8Zfg+3Kv%?JGhXrdZjsWsRS=^>R`H}yO4>ffDs(KQVPsQB-2ynwzOQxasN zbM>&wEB~v?)AK=#L<(PVccmmowaV2@Z9iOB+`32gma4|A^~2d0uMatt8!ZL9T!K6G zn~V0P-H5%c3Z&PE>Y8vMiY8$bedcW2ll0Dr=){#HUs}+$OiE^|{0T#k7P_3>Rc;9s z3`Uuh^yTLM20C%LfhMyRvjByQP{fD|bba2k52)oZ=W)Q}(o(N2o`0u~rNr9A7rWhc z8TCB3Qd{u&XL!ST?VaD>UUxgJoB)imrRSWVH)o(pC(%LO{s{k9-F|$MpMI0C(4oTr zLDE}g0chO=D*bMloV4Mr({RT5+YH}a5ABddTJqS)`=@hKZ+BiYx5@YkPn~?*|F($$Ab(@#|B!}Oo=4=WAp*R(8 zgvaVbUZX?z|Bit7ef&Ol;93d(OjYrx&-%@tzxywT$+$W>YTR~FZ!I>T<}K|L%zPcS z2>bbvdRs7f2bk;t+QiI2xAh?UNH_72&snA^W)V8s?0?UDrm;ve1KHxw4Lk0Wv2%Be zHLT7?L^YI6Nh4P|g{9cB0 zUBCU7in!=pHU8x(>FTC zpSr>2oQY$sTaNs-;gUjqn!`%m^!3L}BnCv%AgCPV1QJC*&B~JBQdRrWCq{szC#TpJczMaeW-Na zW(MvKlXVcsewGC3dzuO7poGnksW;HUWURq(hEo_P$t|aB} zPbKH>{azKryRWMk6r(EI8l?h}odEfIXcESFIE|#$8`GaAJ5)>k$}r;)C*kM7AoKk- zrKuR3r;}Gz4y={84;>Ih(5wF+nh&o4^Wh1)qaB+X=u8DpA?m43i##;xq*RSripeXJ z)T?HOi8s3_#IP@p=Ck}dW*}fUq9COcy0Ke$o6*s1DY0@ZtM4X!*2(q!Q8bi$@i-^QD1ttU*MyTCdBDDP>zB zS@yhW)C5I?*Tfo63w0CfjMyyA zD~odBFV#UYY#GWikb13$d^=u`)ybUCWbbxxsi_!llu$Qt-utah?PeVM?UvksG+{pd zXCgX4u>c$3-`8MlKSfYmFTAo4#k^+W;$Y|HGIK{;1w6@9TAXDPp4)?1?#~L>)&76W zz&G}X!OTjLx@<7cDJ&=~JW`$~4PD`l$vTsK<`C3)LC(LRJhI<> zj_6o?nkipz;eIMFwsG2}wYGK0ZU(`tG-4n}>?eMeiU=<*1xB)P;k@SVpbzTenro7n z*V&eUgqFNw2@*m7N5}N~m9Ci%U&=;|nz8^j68pJPXnN|YLmsR>jed%u(V2r&rh7uL zOfu-PJIB{8&BC0!BlGDB6+s?Bl5H;|neAgjVdq;E=$bwxO0>DNmFv~c6n*RCMl8h) zHbRh28C%2O>Pzk+2|sLmH8*2J!_Wkm&9;u&LAgHCI`Hn$ekSMB;~5;nTXn#0;Mq^9 z1zsF63IDmKK7`N-l8a;wbP}tu-4yEcDqcX<&h=l}Vr3arHqBn@VDNFW=}Pvoz^bm| zRzX%h*2=g`+q!w)HaB})&q>Z*1!DWwHqit)jB4g3<$k4+(IDC^Q05Cs{#TEkK6!*Q zK6xwI$e(G_YVMD}+VFtFXr;E#25!tvFAopvR4uz2R5x&#u{Q9AoSU5mc>p@D@&^bc zUGmW)X>Bs6&M@9hn{xTKMyE(o$Gf4l zR4y_Y*eUMESvUKPDr+pL>*zInT-|J*>_r7c^un-$O%S)|cKY(_HXkw>SB<)S1D5pj z@7AA(-*Ene+T{d+D^2F9)mJueEod@E_wM9ZlUDf5J!S2c)Zt5KlMf$g+F#xR|3XKK zmEIA#g9!zVty6kK@MJRD9Ob%cj^CZ zSd%PNQ^Mk|#6&t9D|7p@kl8h|exgN3Mv#gc&DS}4wj$Iouj&T z88t+iZ*AC^k}rqOaLL-z^r+0pz{zc79RJsH_e+BP?70QfFy!?cv9&Kd-7y8gp-}9i zH8KWFS6iSX@X!SFNvSqFUr)U}a3i9z0_TuY3ybAHa4(`Wyq4|vW5sX$`{<48|7h^jG2j#@&w)@H0!B=m9G_1R}>*sJ6_ zWt$G?Y2{-`t4W8!B2$Id^kV%&`94m~sJG7!uAr*($lie0-}cjKANYnbHmfm2fD`zh z%kdlA%=HNH2GWMJ$*i(rkh59TPYo;^i&?zMR}&p?2EIOZ&487%9yMp}E%Cw-_rhF$ z_5NkWfl+>n#8t~MYNC(ChuEP%KSPRnCB)sga)DQoySE^ZXBp%zT%o+Kvg>8u`NT3y z;j&putMXyj%$W13Dbat?5kKrK-k@WctSYoI!IUw9XW6d|ss4*$r!0R@q}(F@40<&T z<7Y|Bm9JsMw_fue`-kOhGBHuU<%x1M%t!@$fL2exwIO`{rEaZVVkYr`$o+@z0gouiGS>IHmGR$F_#sae+wJ0ydxDDK0JY zAU8GfSHG%zxWOz#CHq2Y5U@5U*<`ZVTjl4wfyR8hbailXUZAgjH<9|BVd9C8M-Y)e zhn?9c)!=tXVV5)uHYYx%fLk{LWlYA&8c1?c9k*iy zeLq`TY@9O#_q`c2R?ecR0FR)kk1e4KvJ6_r*HRGA@B3*tD#(M_0o!rkB-Aot)mWr! ztNmirs^2wy#m+ZE>@q>2d(y+5tf@FuJ4vTb7PlP4toO=2IObR#GU!gI-%=VNwCKOvjuYr;NL~_LQ9}({yY`SmVHJ1wPTc zrm0_{BJ%05)$>3+a+3_ljg}lJnscRI|E(7$84Dhb0d(=-u!=o@X+Ek77OkbrH1Vn` za1MdnQS`)7z}%)^k%il)ENKi^NY+(dy4IJU_!ZuzdEQz-!=<9+i)NmVXk+$8)$rMi zoF<|$oJ);KeuDyirsNFGa>|QSYHFCsq z`=f)S+{&2q-rxKIThc?(=fh84Gl{wGemeh-n3lFm+cEOrFQCVB^to?>^X+l-uRa#;kxj&Y&*J@y97`4P=mdZc& zHn^m}ij7OETK4=#DsDy#nWfp^sFwhBuHKjh}m(IH)-PDt#jAqM>WfIcY<7TNTWKM4i$gjv7%c-U4 zIR@UbIec8SqrPRGk3u5ZMfVYL8Utkw$cs0_LR&56=z z0A$p>6vaPvsW0rFC`Vq{--Xbr?@DV0$C^2_v;uIxv9FUhcY3o^=u(m%4>NH!+`)@Q zC_x)?6NZ2GlHgsoX7^0mt;0w67Y>(hU;Sit(}nB~PSV!?$^|AG7j`av z#|Tlx3|g}N?M#z>ohhS8+?wiCUS7GHsgDWuf9cq?jE1ab)NVSnB~+-&~GDeM{u!_J1<4rxh@S(y|qFZReVewRoH>^XT;@&(Q}i z*YN9kaywS$ndJX%XDO?#mtgYt-=0;6C+=Xf12CJT`s3WJfsEdnS@)*SjXxtQJeq&+ z#8L)nO30d--KS(eM%m;>$-+g$pm;d*(tLWvtl#Ompo;gIU6WjRxX_3Dg#_eew-e_7 zF!t7AQK#SgI4&v%p<;l50|kJbuz2Po&F6gM3zHY zO^AnnF?q#?m1qUO-HuzB&1eRV`>JySba)A8ZHd}zQ3xzBW>Ab=ShcPn4Ciby5I-gI zA^H$3QKR4q;4RxOtcNhQ;|G&|Av z>ghL4XFFZkHcVD=7uRZrsRP zvlq$` zIs^GiLdx}c-hn!WD;r~+h2V~}u1H%q>UWJ+o#9RIhHlRK>C`_E4Yd07klA-Zrv?y6 zG1i<4!m?#~x-?lXyDKDpSn~_+cG06QJIvsOO(gCw!TjZb^~ZsTq1!QI z88mev=nc|`{DvVq#}#j)zFE6i!`k8DkJgEK+vPS?ty`eD-KE7Rf_ zH)RpR6X+(~?M2 zUj5fz_WU4?=UzYQ#EMAOi6$WDX%fj$XX>x}DM_O%Rr}$~*d^p`%0&wvI@=j^Lc5z@ zo7-lY4sxXe+l4It7`0>{14@38Q zVUH5dN6+YpYH0O}D}YnxKaF5R*zcb#GRkNtmBDB_dHv58I2LnIh+WfIay4)RDzIfbB?=ufOsQ1!caIMi!DdpZ`BaLeo zP7(z#Rol{LJWvwa21t|oAgmMVr@$YH2*B>p;jKII224VUWoM-VC(O`e17;ze2fpfF z@O_JPnDgQEL>oCTw=XOup3V5j3Il#rEDU7V%P+5And3Bli#*E9jDv=L?j$Rl%=OE- zJ8Kzxpqrc}yj4j_3d`m)?uPx0ZRmGL?S0R`xVCtaI{nmtT)978|Mq7Y=J5}W=4_YO z`r6{VJMQasnr89$$2Gg9M-O70@8lbTON+^N6z{wrJ?lb0ks00Z6dCL2VC`EJ-v#qV z_je|UuPM*b+GU}!VwsTeT6ir5Y~IXy-Tgju)k!GYv~fm@i(AM&~l<6zgk zJJy;dx?B05I}ocFskYrC+%QESc=)iy8Laj4rk2zA43o4?Z$}>n zlu0eA;f>v$O~6A^)XN#`?uyE)`Rq&|QZ%+`T3Pl?2LeHo+M#FOM?Ufa@1<H&Wwo;93>ffz_C|iCri=Bag@$VuBAZgvwi=648a2vIYobj{I!quUO4UEmjWu1o!_p5N z=9MFdc}?$eeL90MUQy6SjMNwW|A5pg;=QfFqgC}Tx+m(_(a8CdxP>LMHniZ<$L-6v z_T$*nzo)L;^aUFh3wxl(LYX$HjWK$A{F-~Xpc?tOMTYk)UK+E{04DGULVj%8N6VTs zm2O+LNqd^TQ(y7Zh3&%)Mi^FA?s2iB@tKbTv{>AzB}3`HaKDnqN&6R~p0JnVq~1-Y2q#!O&ou zN3Kh}%91H_l^x#083h+yi3w^di=C?K8y?itv==Co($2iH2-E|ge>XX*vwS*%)6Hou z9ajE?jubsUp3rEUR<6z8dVf^uvuG1>S$=IZ3hTGvbcU1Qu)e_vruih{gX9G+Qor2m zfed#*Q2MYf5ItL)|Dce$cHQ3jX!qp!j|@}2!LsU1_2o;1c1g9#h|qv~ehUg>`&Ae7 zWWh3pj4$}dcnPtcZ{{3D@G#@c=R*5Fkl2q7u5!*u-d4y_;7XEDjEVv6mH>%VD=>-5 z{KqySMCWXs8&VAgl08UaPl??AUJ@b_;(LYW)B6OEg!>t)9geq!GYLXsEHg>Edm))E z%Ha!`V3LFi4?2Ua*+z^gpv0TPF+o|5Om5MAt^P6NS`d zB~P-*6V?2v9|6apcthJaMyi^@-Xi>6Y+qxqOcnK;6Bi5%O2Uf3 zfsz}raP^a(CT1q0ymK`;62#6=AO{l1bm{1Fw-^iR-@W}hLk+#bk!6ogy|`v`#m6>8 zIP+k}eGyb`c}XMHUca^>o@_}BRjn!0Hoa=Qd-$DWvC1x~Qj$ep;q^+~ZDmjz_yRib zVNL(7#S#;x1Xtw=^4wYRFX}WneXIOOX%AdE1WJq4pel?%LV(KeV?v{Q_DTe4)pgzo zTN+K!{KM`Gpg>xaH2X`4t@?4?J!d16DD%>*;Z^~q<&rUH%n6WS`_1ykgxx2bNJ1&6 znQ4JnRsJJvfg<(mjT<^Tf9emmCqKhYUIeS*_T8IIpKz&AZ|dKfE049*Fi^r5)^ZjH za}$qIlz{{~=)Mz&y>rXP9Zcf?G!_3Z-aWcycT5==&5CD_5%H?ZoLUvhF1lKS65H=A zjV1qNoaSWhaPh1L39{{K{}eXZ;I(*d0*= zuC+|EZ|<8;%zb_vR6-(Bn)V{@^c5c14VHx_j}>*k*@kJao2z~X1~DET*KQekoxL_= zk`f6Zo!oOzrmIZx;~{NYg#?>opr32H;*PvH84(y1P4BX!mLsDC=5GVU!%u&-B^^}3 znqg{zwpQXhn#9FUvCXHT&2e(LNS*T7Uhw=ovf{RkWkdxJEL%opQC!$3) zCh7H{l%I(eHBX!rCc`mjZ1)(97hHzs?RY}m$yWwm#rL>5r?4^$P@tFh0GwV2xxUy_ z$IfZ12}+LOl4}gLHV%4PcP>Lf_w{+uDAvq017>+Vk8@;_N!9rLS%JefUrRpIVC=47 zH_yQa?J)w>&96n(-k6KM=qt9De2zzG!W;8Y47*QxaSh7M&2aU5d&oGnW`K|VWG&<} zEV%%2xx2-pf-+Hzz2F1ue*iEAQ8Tw&qnyZ{^n}CO*whsYR%#Yg!RoE%@w*{L^t83YQA4c~=DV5ml31vLYtQqwZP;gB)$3=*XvD zyK52EWqc!ZAUQR<&k5$7$mxwtiqI9M##vqAeMYz^aePN;<~XpJli5^7FTAMPW|uwm z48$icn>P}xHg3itEdx=vUJ$Uq91C!9yy{}HFiq9_p#7!1CEsVwp3x+YcIc2CE1{y8 zXx6tpTdX)pcatmj&vA|z)kFGRNVw4V6OkVq_VVFcW?kQhDsM93-+`BpT~++9;f~Ok zJ&!L@+y9@9UIe@$2gE(^cRbuTkQIrjOl&jta}q#g&(=6XLY-KNk;?h%uVxmqkV`5{ zv(~)*i20G4EjbiFr@HRq&bzVhwj#A`bsqFf9i@7MvaYd_2on!Pyh>Ya0on1Opip~r zMu^}#uoZ68efncF&s+4OUPzEdzIx=SXI^sr8Yh+18YMSXOe#s^?4~jQ#%;<%8xi%Z zWSXk|pfy-uU&a?wr*Ls2?i>kujdHCEZfNl0W*miVfRnzOP1lQ?-w$+?*Ze^sN`y~Y z&uFt@m&kHHTVyPeU{;1<-ZZa`kD9B0YiEFf-mAYR-4I;%{Tn9>rOo+WkVwon``Tc_ zTqbvvY1`gydq~8#9!rS<14?XL!XV3i-mKpP<~~38hV21(35(~LX&JaU(gn5x37HwN z^1?Q2WQfjybD~vs`V!`NV;EuK#E^@$A7R;H>z>Ec=!_i&ujcLFyD#uZ;V=v3=?w03 z8hKr~*wsJ>AXspoc*b#;mwNl{6q{~GWNN#H>SdayhkeLIKNujCsfo!huq+G zyP~HY3j$bz)3D%T4tMH-$kw?pGt;e~zD#|j$a3w&&smd(EyvwH{BG@_ar!^$=->0It7TKwIQu4^L3F#-bIy?T^ENfbeK%2}DRc@wQDy*@H=Qtf594o|7tP9YP?ubU zS8QqaI0+=L;`#=x+||k9o$RV@3xm0<3NQ9;)^oGRzrmxCPlzdDjZGpY4ismSj500~ zPW=&URC-PN*DynMKBXK$-jlxM5K0BbQL>LY_55+Pa@^Y~_R;o87Ilp@W|+Su5L{XF zxm1;2yJzV0(u2M(Sd7C$6+NAuTFAqB;biyfXmnrpH5w+(81+=)O#ed3|Z9NrSk zO07^(%n8XI;94?XK%<iaZoeZ9kG z$oF|}9uRFOEZZWFMcyG{(yYo`A??I1CgX+_$2F?8XQ$7Zrv{y)is|!jc}H^M!SKTj z!K=9`+`=IQvvxgrs#N;ky<=#D9l11SS<#1yd|VDt3s>&%awh{km^YfBs1mg+_TKoY>CUb$M0vAEAZQ<&|9=B zubM2N1;etW3hw0hoh?fk#1^|DgrbF&A_Y!SQ9~x%WhB|QTXh7?8rROhc|z)!6WIGY z_B@3t<9=n#oIHCS^#xYEx^8=;#f47PtCrefO62FcNu?t2`gFl@?l(Vfh+T*ai?E;Xipm{~gMIQ?C2B0+9k|pMoGgv{ldEyj!!}8nzUl+qpOyM|E;c6dh1~$#l(rBfm&{Ww( zEWA;46ROV55@DQzZ<*4Fnb0i8Uyy^!$d0eE`jjcDBc1515&;6gP?pc#*|288t=f6b z$kfYAfM)aUcebVWNa}joTO)kbIsBZltjF#>_3!C*R%YfZP)<1Wf;Omap~1A<_fu+C zJiDTKPTmEcKsv;2Ns7FbXA~1Gi%Vs}ub&w`etI&L_HA7w8A@BYgKssi;QE;hexH*_ zavlM4#d4g8Zx4kg`8sI6n?Hw@EE?X;gSnb_>oxN0$qtzz?qrS#ByIGrr|us6AE@f@ zvC%)H!gsYLzPMQ`;hbzg{(8q;9%?6@s;O0apgUKNlhTn5b-=i4lEuF4JzEPuRt`j&^B zgRlZ9KA-31&ZLOoqd0?1F$YGGU94I}jdB^+TeKdtq`ATx@AiN-(aW5}qsTwxxufF9VS_ly$*R0D5V8t zsX#YkGt;kbL_$RPNoqgBOKN;>%sH|m71<9X;sjG#Om;NJ?LQTY+DddLvZM60s)oC) z4U66S(^qM7VYb<;xCoyVslmMw!u_vwNA4$NHO)Kj91ZAvpup?c!}gpB7d(7pJuRH#KJ zz^20{Kq`zRIK|g*!iwz6>Ecc9C*tt)|d-pCEmp=ty;Z3kgQMt7#bRa2n!|iB(|@4h+y@!UdsTP z{zTe5y@d2sGItq$uN6%iXYeSJ#7+Fk>MA9VFWip@ZiKo59sp$04s|^jz9}QiEsT}{ zorEr-Ddp4h3W>ogJlO^R1e%wk4y}7V4M6b7gNK3(1QycA-+FO>lm0mJoL6%{z7ADi z>gge~Xo;VNH3~L{irUt6uJ}u@OkXJ@$BW@hpBbVs^DB{^m@K!%KIs&R_vB?tJSaza zr|okm6{m*_K@UcsIWWx3AckW}P8e!iBt;%Y%`Zk?u)AA9Pk5F7Ao}q6nS-b|70g}b z=0!P7+Gopz=UGm2MFqO*Le|w; zK?2ph>!#@ADal12tShPl!?##j{YtFOLZvAxf{r+uJ<|pi{;(P%26F1S#WS9j*n=4O zZDL4GKK4#{qx>)hk`~yL_u?eZ({smJ*yahyH}8HeUKUnfoQA*WM|!|PWfGcnd(&?? zjTk7Yo%lcapM}=Hx+5YyXp-@o?Ym)B zf!iT>wK8acp!IPA%H&1g{-~MO5pf~a_{`O8<}X`$0b3i5B6@dgt?g66xNui<8Hw*w zUq<8EC;D%%uX4Y_y{meehodzqtLUEUAx0gJK;}`(u4?w%Z36OJ*8WHQ6QVtR9aD>j)K9%x6tRZ@t#J(uR^Uc-*K%MuVy1b0Ub zXbpJA%>d4)%3u!vVjafAlciz8e?X;ccQkvi?vm#6$wdG}2qcfTk*4oOdL)BpVkF=0 zr0zY7F|shR!qGAAC8H0bP>X(b6I2&7uhr!cU)JOnKnCje?D&hco!;dZ2N#kC^m1+xqM{ta>tU~V!-!#dLIAJ zjgV0I+WXK#Q#z^a@%nw)8s+%i8vv&|3NM~6VOlG(5c^g zu)Nd`$2p9Sn)Oh~ojr=c-T#|_a}eDKz+EbQWrfJcadi$_!+#?@LK z@0F2&sv287cI?RCyeCntF-mg2K7%`TkLF}a+6~S=G9seu#|dyOcI&MX8wbHmw9m*J z5v_(j{xo-QT#!9DBIOjo_vZ+8#0$a# zS79MD>3PCOX9e_y^nVzb<}acM#jVI8+^B5!;zplMzM;Ecz;oEMEt26MB3p#_QFe1j zbHMEpGxr zYT^Xg>F3w(UHOqqkNU;`AU??tfqHvNMsFl-}<$uM~N=$lo#&&a*sm^#`p2qQTE0L{M7rjhx5ekb7}v* z!l{4Il&873DeC!sk>NR~_}p?@-RX>)!OU$c#mH^fV`;=ciBCk=N`Ge3Ubs7glp=w9 z_MDhd|7vWJK8;BuktqSf&TqJX)NVO`(d$CbNEH#WqLEE%@w1bw9_1HTiDW~Ha%1C>P<jgfvgl?kn{g71^1Uor=8+1kX}Dk<;io_T{tA;ff-f&ouEZ?{Ok(Q1V<>Bd{pYdn z150|=duJ3%^1LVB)N)M_>cv*!5Y5%F^3rc zru6qu-WK|E!CXEzF6C}F%xf2|sB!T$0?ZFw=Z$>m6mt&oVd&h}u7)Opi^0Q*y+?fTTy6dH`G(Oz@-YSHdE*pcigPRV zujhSr11G8j=ksGVo)o#X|FJZ*6Oz2#Pc3LOZ4%c>M%I=KiMGT)4(+F~E`w(=cp&yZ z-mu0sL}}8Mdv#-{_B7w3=C3kDPY$Q@|xt**9 z8ZcPeYhpX6Rei*JjCe%nT1<_IhuKq}aW`!a;?Ao!K>kb~|;|8ePYh!+0wr$ztrBAl-`GywiiF{)4-|8AU*V-!b>Bj&-{HlO~4 ztKhwG#AZ!>Zl$zgqNLu|uzZQ`x#{!_rcLjg1^=Q<)wMF8?f{QT3-Nz?3Q$TlPHI2* zn~h#T5-xvVz~9bJ@l$D8X2AT4d`fEQxw3=!EDNZrPv-VZj`7cQcZ|?9(u!T;ZS?i< zvWNx)g+_@~ZcyC_nJ9Q{o?x@1+$1qPd3}Uy{4q4)9RBwv8psgLN?)AD(>=PG zQ3%kX{bQe%>6hwO743e#9|2gU7laNY=exyHM6QT>f2gFwv>W^ixIQXV>A@O()EsfM zA^kIH#vNzNY79$a+Eu$ED)0&FFPC~1HHueF3lK}Zvk^9u{E%AiGNt;#tbyxcwwVJ9 zQAk~GT4m=1&bB6nZ`f=XW+z3H+mbOaFe)xpRpj)-ymDHM>K*W>UGpir#8Z z0VRe9%IxEuBHx6v%Hux=R}xKmjpWs1Wep;-A!J|lx9Up z&zPPQ?i|5Cu6D8Ke7w<`5yzfx9M%y*vdSp7{@i4%L$FphVYbfwa<9l`b3-W8n_q#H z=c2ZH3lgesFNwdt*XmhC0tSGpq}DYTYyl-2(5@D547~I zdy-{5=?=v{flx8HhtUs z+{jelEc*{juZ;r{K&Q@LG^OGsUy2D?2m3D--{6qdl_vE>zwQ*9@U6`PjFb0UPzQs za`s~FS$Ys?7R_jy5sCjkK_*RCc(MXAtqD!AdSc$}Wu4tm@iMiLxxF|hh^K6oBo{K` z1DV>pu7sMj0&Y7o9f!LZR2R(|{>!421#_~S6#&bsDlXlbl2Sp$ESt&NLoHY{gBGa5?L6=x(;yEe_QEITo8?>T6Zd5bSylDnh3wS(D0ZB}jxj@FaMujF z#wx@P4P)WK$Gf1s4Q#*q!2xN*tkIF4rKjxa;~&isVBLh+P2~0~Xc2_-jsRnUjgV^b zu@DPMiS2W$gVh%UW~(V3a-yfSHG1TZ}4{+#z$jW+4wPS~5T!*; z9>NbGWf1M{m?m7;ZoRV$9TMx~;^q9s5F&Q}ySpka?A(~L>_md`GV4GQu5j00btN+^ zxQ*y(SP;KFFtWHv7Z&*?G55I!kpc!WH3gK@hn=1quy@ba@VW`sr%N1yEKuL#geVGK zqg%-@`AXZ?m+U5w>`J_>g=Fa9c@3#vO;QeCxts^LoV>I7@#>P!dx~Ovcl%$U9?;Iz z!w~zii?F83mcr9FhD|O|7jNQQ;*0jFP|o+AQd(X_lfo5^tf=eJv~7GW;kK#L-qD=! z(E!!9oyxhuI^iGfedLN6q)w{>kI+nV5Xd zbQQ7P9}(c9v2)9?9jEIm&8x@6u_)}1l!qCv6+8E-Lkdgk&V9TmdOP9ZbCLgnk8(wU zZpP;8er0R_-ACv^mt5qd5w}&zwFykqG!;?rAGNWVxB6b{Ut&i>YLUgg-vP;IRS5`ACb@Nt| z>#Rg2T)o>q4#%h+-(!~toO~m}pEo>@U-eAMDDe9sc8<}_RIM ze7S#I>haH};Sk?AI4Mmc@bedv(O*QF!r|NncVu}QMl&2)P$oOI>?HcTmHQud-fdQG zj*Y9dZC46&r_x89SrpoLXGQw!6x?HB?VkIGV&Y`gsbVYPgjKX zvnZiF-^{PHezm4?sl9y(Z56neycY02=^g)ktb=>Dr@MZmbx6{$AO$^@)hM73Z+{5* z#4TfKG_e=@t@#z1IRJmfBx7k-i$Oj5f%uMUiuJ%KTNs#U7&~%{HF1u=tU!9OZ62R+ zqXm@?Cfe!-qsttIO{EO&ko~5Pu=~iW$y{5MQ_i$oim+lvu$m}EDum*n@5EzMijk{f z7CQ7hkm?Z;sf|CC=_CiW?^KTnciVwowkfSZUMlPoEIgd;ob&Ix=>Lp#!@2K{&mOGL z4_mD(@)@~SkUqXb^XNftzw!VFj%<9K)H+K^bC07#miut8yunH2iMIq&Oe$VB z-yf|u`qdo;|2!gPRjV`Y&E-FLP`A>23H_?!mfy}dJjY=X_IrZgRw^cYX~@e;oIgf~ zLfhNLG^t-1lZ{E;0VD(sO=!oCL)%x5gS-DZoXbGV{AqsEz`KR=7;*Zw31t*`zkt{73cH}8YjhHIbl?5TCV40P$d=6Cz?`~P)Zz4QEs8P%LCJZu~1qRC0J)1JgR_l-QW&>3hNmTmz$*ZFp8ZH&Ks~zz+trhb|l_yUw zRyQSy*B!J?^Ah!bB7w9YVu0;jDW^~JES>F6c(UXTMX+gek)X*-X%qS$tytShO00#2 zq?Xh%kP}nS&5uf$%-0~^BXuY`7A(b#x^6wfsV^m05_ga!z^$Ziq9-ykI1V-cb5Li1 zVvw!P#Ga9m>e}^n4h?A|;>kdCA~id)J?)HHbbg&t1Wnfj74tN)v?7$*!_)oy-baH~ zkCQ#sgOn~l%`M>EPUl%aI?#Skn$P}Q*7fM2$e+64$0rgl^dR_BXTK>JQq_%eG#HE} zWl^UZwB`5R5qg$wRs%D;P>sb1CM$ z()*CvKho zbdYtJKPKzZVM1s;MQ3IGPIp4@+#Srp4tDS?lmi@wb!B0i)FpMVQ;?DfrES`F1xenoDH1=Z>$iGk8>k*FN&~ zXU4``Bt7e_3Gn9UX0=~_iId#Onq%VWi?WM%hqvzLe?3a!!`=h10SmX^xs%NLhCPjR1;=CqUU)>F13jz4d z7!+g(x$Q6!Rh_{v<~fvIVo;^KQLDJb^j6=hJQ2j1;cq+ZTq!-xP zz%)8>P4D+tFktveW9f&->sEvGxt4Itc9Zw2M@8O+ZOmpO1`~On_!52B8{Oquvl{KT z#3~*-=?i_vV{Q5xoxa!+HAW5K2`~%nNI4`g1a^4JS_STgyfVyW zfQ>3>Lbx9nj1JLLvL)fafA>h4XkZvV)wd%?cs^NEXc>Ob%d_ELmShMI=C!f(Hc5WJ zHGlQrdnn=?BP}9O8TFuTo9{(BF%M;YQbkKv6sMJUOF4oDAX(9zCZ^VgvxQp%Vce97 zEu`~T>Q~}dtudYkfq-Ea3hrd2?^P#pW&SL{H)?XmLQ@A!cE=C}m!yBv_dyR2NTL&e zKx9oYibLVs?O@~9zQ(bwFxMKVkkTqwKsQ6XN^LC1hHDBAo5ssZtG#EE^c9!tS)UN| z7I-;MZTQ$q@uDkg5?s-$fKVg?Hmt+iCTNw^-#E?XFt4zJHo+FbRvl_6M<+Xn&zPCt zFZsNB??S%=HPO`1=l83;z;H!%0a{9pLEY(?>;R{nFWQYmNx}R^ZE_`*C3qY|Pe9+K z%;tK;fJ+zYP*9{-C=ZL(!OMGs)1=O#_zwd1Ctdso1b@SyKk{}pYHihVwa$GJEmTo< zD5v-hHH_Rqp2hQ}@9Ic9VUx<_==D+amU5vOWz+kZKB--o;7euX^@E%}PE*#rVXM}C zIM3O9xRTaFYgo^cMn#0J0;K_#*(%jkWr+{W;^LqiJo)^m6xa{y5&$*#*TNDzD3C1z zG*W{a3SH~r6I@G%@ab(|Ue%KUnNat62=bA^AmK8et##St1S1E0OX*{Ag#}v?dC_1cZa0cK?KguW|_6w#Uu7>n5)1v6?Z~BM!+y?*L-asx1|AU z6D2D&3k!2DU|&gvRadQsyWfLIc6e6OrL=9aFcY-PQ>S3(YkX`ar(@YhLpxLFe4QpN zp9k_k{tk<*NvL23n%5qt6ZSgf20#Y_SixVW{0Nxx%MMTbs6extnNAAz;SaY&dK{9x z?UjhkCn-TpydtW=VL`cmtm)=%V}GaGT+1G#ztGjK)~9T8MkOfj zxokRxoWQ!(BPHo-^A%Tw=?BdhO+t!bw%lL#6qyjI()qfrE3_B-&@a2#)7J$0(V4-$ z4)lQr(dh#wi>~L$|BdE6LP_J;#I1)CEI{0m3Yo3(ePNqo1KeZw;f@_pS~Ul19wDJ~ zs!LxhQI3|}(HQb;`Fm)1?Micve!l+=W2dw03BkH>V;#;rDGYYX#%XnV*2ck z0&O;Co3w)|O$!z!^}QE*|ER^Nt1)1rq(yF3er#lk&~8;=5r~`=d#}eJ5M`x_VJzq) zi;!z)s!gD=ijy8po)NiHA*?p~WhtH+Uf4ob3u%0nu69D^5}V-AX+D(w4R{=n7CN44egtMb$8ubTyJw^#8^;wX#FfJIRCknvvs4*f z90{cv(N`cPOC<7}516%V;nZ ztoM=-C39C*QwapVQMxU3LxFmpdBg{OJRX+^cLR0NCgp$^#TSLS={;h>*Z?QM+8k2Z zpJT{`Oye1{In(MB=jRzI2PtOQer3Xr7KDsD|McBI>Xbxol>MHj78pMP*9zNrieUB< zVQBkb!RZw6d%GEjFDJy>rBCzAFZH+|j(B|LQ{8Z!00&glbDYv_YAOz3_KtD07Gc8A zTqK2{Jsfa$e*#EV0o7IZYrEf{0Co6iOpj(!XN<;(`%z1X9O~;q_r~ctz zd0ViS&p>0?eh8yyLAGKwdP}*&h^6)IM;7^9`>R}7=%l`Ewa&nr`x!tzx}FwTjG^M0 zm5k9L|0qo@t%K{rdyIy;mj{N&VC*CXNdT4XRuq8x% zyPK=Qoi;Br%C9Qd^1qy)A|969OHNyHO@YX_+X^L3$K8XzzjRVQ10w{c{8fyY{n4KH z@8dYG;O@(2^h3e%w-vlljI16RSo^BYHY~kE3iSR-Db3zc@vD~G%^x~E$UhQbzNY>z zu$X|`2`JUJe6Fs?!;%{odF)O+n=W}rASN%_Cgro^1kqF8zYJu|apJBtQi2f<$ORr! zOk$7Mj&m+>!K@&XEEifFJ+>(6JlHVinn}+#%Z8=>t)&^XmDaB3LI!kf8w5&Zn(ndx z-r338Xz;{RppLP0xoiXcq_P+hNbnxy7S(h!}qyncT)S6@ioh0 zJoV(1%*_{(wD0!~L1t`+i)VJ+A<`l>;cq(!hT&`!B1vgB+T+fv`UlaU!9QxH9P%vk z!ojt4j8^#IZRQ(o)XR)w<^6GqeJFz&<|K#p6?uyt(_4bwcb)gM+Y#Esr7igG&6;~m zZ3}DH#MbjLYe1gEh~dZZP$Ixjn=es$$Zmg5^S>c+d4GscRccV1ka%)7C+z04=N2hm z8SeTsogk|LjJ$aG61YcvIuthrRDT+{#1-HQ-puS-c-Jm`Sk?zd^4T-Q@GhM!aBD;_ zXnqz_(qNIypb{w$Y608KXSxH?0b4Mkib1zW%3&ohoSSkPP8;#$Cq1HMMy}zlav@s* zOi#xzd%n)!GGl z2mXQDfJhg16pgU@3ak~muXX!MOOi-m_#hk!T&rE-dT-JnDa5WbaFwsjtA>e>P7A0$ zKmFe+cc5zph#3%4N*Bj$G@dYgznabLqh97cdQ+BFfCOgTJ52z60Ji{NRJ?#?2hhdu4NGgM$(Gs# z%8|`mKvg&oy7pSU!r;68s4x=hGMa(8Vpk~c!=7tgf^D6~2}N&ftj^df6m!PVoTqGR zwcWePoG~z~(HRoBa=z)tXlORJg`Xi~z+QpYVo=xfAiPoW%0&?r649rE!`J7h^hq(t zo3@v(BKKT!eQZGf4?XT-+4&zhnPQ$IQ64;hKLVV-${r+vIZ?nWh4Y%&Za_ap2gB)G z`u3Ey+U8|lqzcrFXAK(h-^Ef6o`fBLTbDXe@ANuvtHlT!RPrld`#bsQ?-CYph0^hC z$dxQ(Cwj^8aNwFVV}v5T4+a{U5Ao4-ZFThzn|g&rL^$0yqWzfbE7Mi27P>vHkr!Oc zgNu?^(sif{6hcKeLV_havnN}uO<~YZt_<4I_s)Y{qtO$!DGojD8i*@C`Bu1TXl_o< z)e<>*Ayug#4i*fh;s+!5fG5J+?)CibQC-_Xu?N!GhxFJ?s2|qxS-m?dtnbzq(s5;Z zqI_W=N3+q^wh*or_(TsWxf>XLT`yV5zqB4Y`m$o#Kd2A`_V&Kehk*t$u| zGrM(#i|=BD(O~NR=8_hxmg)UIhUvP zm#uWJ3PEm3GuysbbZwUD=j=-~?I6@8A=Iz(-x=hatPGwnvSnZiaeCO98#Ml~7F}g( zGk;SZX5-l;LvPV!YR7LL_0@!nOlxg4pO$=L?A>-Hf+mLR;G_ENp~_to0X3=d)mV^A z(jM$;D=1mG)aLFB>+vv}s27GLU~~;yt7oz#X=Z_n#LJhN-X5|YCk_WxJSP@c5S^1H zwwM!{^plR)Z702s*^W9t!&l9mmmSTwm24cfY8PGgYH*Nx=x5^YsauOR2n}vEF>No% zg*)fP+5vQ;0z6|M8@tOE)i|8u!vFf{fMNoLo>$cznHG-*Q?8JCkC-6o6 z%Cbjrcd<_X=Dyj@lvJA@K`-nC60M>twA$bXZ!)>8yuVxA+ITdtE7+>7FI(FK{knmCOUN=YN4zpBJiv{-7fxBJ~Tah`QK@$ED;dM>ypYB9rW%P}LCUv~Fl4Y#Ds z>d7txcEZZa2);d}=UcN`UuGD3PtiC`?#ENF6`JT*O{&V5kDk3!T zCYn_b^^K&=^p=%j>2k(Vx)7>dE%fW`sz|v$sCj3TOR@)^tm{D@L@2eGZ42Qz^l5JB z;4>u2#fvM8ZaM{YH8@I1saKw~qPMeHo)-o`zD+%#;51tf1{C(^9~%}7KP%!%!0z5U=kERrkDuvv(DXRPb;4vh{rKJor-8V{bQlb2p^5 zb;-jl&1M+M_9W%o1(w#!vEYF~{^NmA%rOFs%=ZcRdHKilJ`yVN_a2~hrs!!3uH{5V@dU{~2V>sJ;7W{eha-KcW z3l^P8Mj^#?#(B5-aXM$1Gj*=*jfv`+p&WxawkykzeY(`+s<^9L$2nmy;-7`51-;zI zeuy2LRi!4sTlq>bOqRAK|6y^0NaLk*xzl_j*VP&$!&JY0#pQ6@x9f7d;d8jpQ@UIF zD;UyWN-Z_$G>EaRV8|D!XvJ42oWY$^=6RCF1WW@{&N9OT)cHqkVYG+!&snJmS~rsb*>S#~{~c^w?qdii7$gp|Q9d3R}w6^C0}$ z^c+j}v|D<~QyKu6vY4XX=nI;IZ8J5Jci9cwxRq<6)%^LA_OgCvS0gi|&Ga@Ko2RH4 zWFDA3-fS~@bs|@xi?xkifKjgBEcE{|_TEuVZ`=1M9?k(hqKJqPX-e;qP^F3>2!!56 zKnPWubV5CdQbQuW3QF%tm#!cQgeJX7P!NzVgkImqbI39L|BHfkN-zYk9J3aR` zy$MPlHLTzM4g5s^u|iMsHez%yT%xbB-pX9T1ma-FYkj3zSlcY4UUWx%g2ul5&-OQg zFY1J@sFGT~N*1DDh6ZXL=%s!aWxr)f@CW}N{)JL=ap6$Q|8dBW;OFpo_2|Ti%9HT%Xcz!# zhz!B_a{9HRFG5e{*6tTlgal*TG7!F#H!gG8is*-m$sAg?RDSye-ziQu6G*2sRiRl? zuF)%PJRM+kP!3ePYWwhq#KwY>VSsM7+A81=f5#z7Nz(uIg$(slxHyGIxBajU{avk! z7e)^*ATp(;e{$gG8qaU52Ww_n*#18J+pPlR`+v_WP3Kt!pl>qD|1F)AF=JEBdm>oJ zJVoBjnxX8-fsqW{jmwQ?fAVJH=c1S9xv<22-0=#_{O5rg{q@WK6lR-_nRi!VqeKDs zYHDj!cYF5Cq%uy;QtK;D*5EC^Ft;CU-=;XRY5(U6K-_T_sa$o=j5`<*fg3+Ys0b{W zwTq6%=o~x}G3oqDo)}@ubaDk9xYJEHKY+Lp5C&~Era~~ve zeUKzrtY$wC7DY8nqAL)ZLSEn8CoX-^2x?orgTeIQOXYI0UG3xm31$D73&HG5=f@g@ zp#pdN)wBQcZhrzf#bai@ug1~a4^r$PFme7oGO5Qu0GAwJlI$6vp+6ahoD$|kyqy-} zva~1AK{y`3<=PrklkN~L{?IgBnukNX;~(0$(#`8ad2Cn03V3hdw0oup0$IOxL{1l; zK*U-x(Fl(`$YPXoFaDChRW!KhBbIlX%P{!nBDJ-yD|=>IS$=;B;+!ZWy||`0;Woab zvuMj*vKB7Eak!R!l)ncSW|?yFilb8Xnm0o!?P*nKprgT|@-FaOSJbihV;0KEKUq!2 z_m`_fh9? zs#cE>UnSZ}5Ws_hCZ4UJ7DSw;{^)oeC4PB4i$quUXoW3%X)P5^I@uZBomPz9# z2VNJq{)C@A?wlr&xb?3uYbUH7Yronfe;&}OnTS<&TIS4P5p67NbpO5Q3(4XqRZ@19|NUE1S!6l#PS-Uet`xg)Ti5h<_&H#9{L{S& zyLV^$3*FAdT+${UKD5{Ean=)oTx*ZqtOj4PXzB(rQMkj=GsW`mV6_U=g$V&4Eyr0g zwH_V>cp)ck?~W9d}(gNV^hu7U^W8sDWXzqwX!+{&5toy#Tm zc$_@_#s7?$K<;BdvZKZOmW3w9HrJ@jq;XxY%{9%`O*nNX>>6L-xG#BAN(ieTvk;|% zu&A}+yJ0T?<2kTDQ><(?rajz5@*`dX2$Hc*TLbB>Rh|EWZ@&(_2Jumma*8 z^Bpxi9X$Q?!I$A601HUADTfYulFY!b4;4=WNN-CZ7b+OsO?KT=(naxF!AEJe zg5)d-+rBGimCbLuar{V60B#3BUvOsc03$3)a@%;HpRt1LCu#n}E zkK(pMUOuI(o!7$92XcBnI4}TBs8ix3>V4ta`DgvXoVTMd&D(S}W;^K2g3uInkQZ=_xkQ~?+%o?L zI1Nee>HRG_n4oduP@VML>?Feqq$dPL-)gIF`Y=vzJc@piSJ4pf>&bwKE~Z;qk9`ek zt6rZczOsk-t0O$QYXtiwXIqY*ul8Kan^Tviq}+0iVEX^{2LF@K+7#=HYfvL%G#k^d zjtQ9&ZdXNLIp&9iTK`ZHcbXUBA2Ip^hOYi?@*kkt>SY*nZ&Qav4OU^fRVrrJb) zL=B)(i5CXE`Dznzu?b-_L|^Io!X8%4$D8FuKke2YPJHB8wOBK>_v=F^13F7Rji?B( zC5Tmk2Tj1&EWe8EJU&{Zq@WW6BtK=6j|PE~hLxHf^IFqHwr2fM#c(EIq=1>@cy%|E z;On>S*}h?xzF)_LS-kAF-wZP5D9D!mCDc;dvl1>sT3poZYlBGJWi3? zl17fWM0#WO4)v0jLz~feYc~zdJ%y)?1G6PgC(UTP%#vhmApASY381Gv{_JUXVx1{& zD#vSuM@2m1^s_^efUevsG~w#aVuD1d6JPy$TR>Hi+wmxU>)vT}cFWPIllU32%3I?s zs3QYusNTJIyc%}-=3{=ZN+(&76Z~k1RTlgvhbnku>s=+!Qwq-Vm+1DN7>NsmtoWOs ziaxzaH2wqOlG`~Kb>%!1To@3{^TJ-stP?4BdNvXbZvo?7GGoi}SMF`E5N6vkyp669m7$8og_|a4pp?00Tgu$x;ibUW@`9 zxQm?xCZ?WMdfW(F+IsQFrJ#gLpW};$-@u~718Sbf2h?Wn^BqiT_$Vj$@^mlBV(Fdx z>tfxv3+S)EjCnOELwA|fQh$jL@JqX&*O87B83&FX?#7RqYFE996SJnoPY&^h;YrzE zYn~9qW8a=(Lzu*1m6p^nA-lET3M1QH!x<;DwN2`1vBpr`@@M9SczD9{-2iB@Omm>c zo9eo*4Vig+$5h+ZiI};vriO!5C|b<%)bpO=K+{*E{wje6 zJq`!P?V1a=SEXLd$t~O`9Mjc04v0FsPwcc&aPs!nnOQ?Pbb|sH1R8p$3HeN}upC$h_p9vXnR0V)GQ&`;k{o63FkS?n{|Z z(X`R^4x8`KTxrmm^E}}Jlls?yI^tsjGcI-uVeIzt_r%k2j|#PHq2Hg9$v4D$47~~8 ztnTXh5$)9dB#M5aVnlGhf}dHo&C9a|K$QL450N~B<82M~j`PF`$GJwzs$~V`aPI!;A&0b;b&j26rAn5e%W-anV-e;@wYlIx-ua} z=D8@1|pC8K@=(orA1$zO?E!7y%!>_ZoW#>XxDQT> z)%K(%qRt8{(fw@Z~;Zp(evO8QpslbvPzpVZ(B}{nkcK)8U6eSzigsvY^odazGysekJ&;kBkw)ayH9 zJBJV8OdX6*eA!5?*B;ld_daM7nfNeO=&<0lRr`Z5hOw+&`>DD+PMj$D8ZnWT?u^bU z_UYpZ$X6|<%$tVl2g0^O!5XiJ!hYc{+PBF&QbPCXjw_?&M>0ZWnJc zWtrpG013_W{2&#p9-k`zYUz2(;C(b^X1riC;uBOd@JT9ItKpx@LJ>kv$nf&NDB7inbUDO~ z^QT5^VOLtaInZYp{GP3Ae%dEO<*H`iI$V`G>lvTJ$)YcTfBpOwirAaxS}CT$xJbg z+^=qv)Cmwm*S3C&Wj^{V?r-7+thv2?v0*>-%ND`OKI-{=@K>N`o!bk(SrHq>`=iM6>=b5(VLO>jDn zuIh(bEk6S7LLr<8UzvwbrSw;kNXUuDeOSpdynA7c6Q`a`IWU zsF=&Os{%x$^-lZKDkE%3pyQbbjo$ucbJb~3&k=S?=D&IgWC7g=h$)gjj3Lw&1 zgiCLoij49^N8vjkEl?mM? zJhrFHX1sF+A#yDX=`+-^XmJIxsM&RTg~pBv4cmQ-yA>w?I?M-4NXCxG?4VvdB%aXU z|9tdq?tymw<$q@~!ob*o+dpR2{;Qz553pu+!ur&7eWKIpGw!t3gA8Yj48HWQqHrGV z$$rIpQT^mgC+NDwJb5ZsYKz=jVML7xy<6dU)?t0yr-z{`a-`O}74IhzR?pa+PQOxV z8@T9J-&xt)RoLlfDt=b=oUzJi!r`(}Zx@S2>~L@Fsnb&^L5y*ud9{w%?78F_7CCXG zDrG!~j=R8tcWpO1{*qQ&A@B|>>$!BJ*0JAzAk1_v#x`rUi>^RXNzK@*vkiAAXF!iz zrzake>oT#&Ix7~mM*;0?j;@4@8_scTFwIC)uBu}MXoNoYm6@vTf{_}&8K@+mv7UZrYiq?2k0Y0Y1^bTnt{X}{sXBsr77$Z-F&^yezT;IjPjH@OWi z{qr|bg8$VbxR}D#6~FhXg2mwHC%y$dQ5?Rk?88gSo;%H(2kjr#+dxMi4ve;m*oDVL ze0FB@Kkj&J;NH~fpyT+;MhPawL(H1~`cN6OA>EQHbE;EjXFfvH#M~mbRJ6rEBb@0Z zRT(>`f=fP16I*kNZOT&(oZcq1^93n~>L+Dt5AUV7B#&e;u_E!hFC*N(k6W05?$pXj z)TS(T?ya+&FGh(iZnsQ`hxa9I{Acl-18vO3grywF2vT!ognaBRe8?hM+NbN8qx2G6Hr zXbxW|EA04j`txY$#is5^;WFi_@)HzA^v;NQl_nwGGDXCROtrC4=lf<<{rHsMoUv0! zbe8N6JZ~|_=&zVKT<2f)g~O74CMjI}2k`i(O&!udH_elUpO5Ho2DU=yI~J5y&lw$o ze;6Hm&}3}9tzo-5PL1-cQ4t@x_E%5j#!NTv2~Uy;^lIKG=j6vCl#7q7IKxOXndgW? z7`dYxWfzmlmq{7jgFL1_Pe#WEF32YL2VJ$l%3{GJ(DEH8`<3nI1BUa`hD4Dkp=2vc zGo9-Mi#}0nt;x1ttMgjr?D7-vb)uYdf7x@D>;eWye90*{^|STUmrf*=JKN9|PVrOr zOkC@$i3@akKTl|XHP`@nmC`IBDBECDF!0S=T&4$!!I(iWUH06r&&nM4BnKbSvDzo8 zz_?=VUzzLoz^GEM#7Es)x2B)*8}H{+QF;&;tRpTls=aDOHFZ3oQ~SmST_@A5KHwZ~ zB!C$Zh_B%+ooi4hQw2EFx@a6=j&sORyIn{5sGl!WMTWEH8?zcQY9KixP%kH9t|=ix zM4<|g`^wFVM>YDxg=z!gC9>$qA-@mUJ zD{0WomHQ!D0!3QlIs;9P=x3_vC-v4+E%nf-0wOH%lQU#)DnciPYa?w3VV|*`8A3y8 z-VKtFZ{g;e{aB|xN;S*Vl#ImaZ*_*(gW?FCwH9Tm3tlapjQZMFAy`0uew4*d<zcSz^g7b*CAGoCE}lWV$-arA>!FKhpW~`Ef(Rg3mywOb;pR z{Ou2|%VuT)$D2W=4g$z`we@;EQKFg;2*@HC?d>0Bd4p-Al{Mxu9VB!j{{)WJ(ZC3+ zs=3L#jnf)$-)LXFgbJCgOdkDVr(#NtXP!07jXcHP94@br>On%X5Y?fj72o6CBY5rF z5bpUcT7`~f6SZ+KGq0{8%5dM(!(7*xL$;$_oMNjFYVfkdLOAQkB_HRr!xPN){9QfN zx_--RoIzDqGm;Nri*ogY^!TeZx&gT>BK;Ds>E1obyv;adT!h<4$gulXW1ENn*+XZx*c@F#c(QD!gV^Sro@V>%tzQFyF5=p#758eeg+FN)$1waR};375`0qh9Ue`+GirOC zwwQPIZ&*ty%WmBw-fOHJgc4>aSbxg6#<2ZoycFKm*?t|m5L*J6OsAdFT-)VGs(r&^ zw3&sAhN|M#t~K%1V2os+2zg1OJKGL1FG;YG z(={K{0_qsmHq9tQp!RY(>LGw?1ran6kFO6AW zXlt&C6w-68K^wMUj=B+^>2)yjRQQXJmR4;3PZwQ2X%nnetLbS+(DA2ITKMF#p5z`v zfc5p_;--i($T-LJ3N(RZZQP&LZ=TY1b^jDnvi1%v%e5*J!NpE=`&k<2j^^2utNO|5 zov&saw~_3dT64)_aBsHG%9(5G60*-Er6N*nd~fj1S*$ELq4XSR0M>QB$)UN%=+Iy0 z!U1Q{sD#*aHc&Uxt;h;a>uYxwVjxoQSlqn7WHUl$8bT_vr;xUSMgeH0J_f7Ruge!R z$O(d}VVZTFSCzSK<>rW5pE6T=;szYvuytVqKB#<+lKAzMgexwmiU-uSKao+2Oj@^EG}ov)jVu$EN-J>;f}n z8#6{k5{}<4C=TdX(vgIawVeI-i<0|NNw;^~CG{*cMuyjwth8(Wy zDcuJeDQ+d3wk)}MV!xnuTfEWO>eQ09xNxfFtgxm@p^$c&FBE4|S71Kul)|x-{Av`V zoDuk!zL+e}BxSz3A8Os6v-_B_>O1jw*?jY~t&1)$j%n5$>OHp{M6?H83G%qs+v|^2 zEmpG>HD~`?wVF00vW+pV%0id|M0D5dhiz0()z-KENP*7%Dbp@z`W5nVD@PkshQG%> zD0ZGD6;;UY!F~hXleMw}dYUzk`!Y%_0J$%W58AD*g5-nO17V{Mm9#v1ZFt=%V5c|h zN~225WHzFm)GOqB#abHfo#B~8@ar#`Z18wJ!I2H`k~$Hm!#(x&7b;M*_1u|`twZI5 zn!?FU78fnsGwx}O8l9%>Ju-I zQw}oV$6)eUN3NXz8EyBtpF5ocQr=4%{{|^x);^vlaN0EB^#6#E3`E*lQCa=>mdV7~ zTF^)qK0f{J1e*D2i1ulkYTsyGyvI);V^fET)=tYOB!!Lwhc*Ew6RQo0gNSMcyS1(? zVw)m21s(fzO|L)tk(%b_*qF?L@upn2B?-l?{nN{!)v_rA8=R|eOOUfjExuj$(o2FZ zJHvXks>hFwri%XX3jnd2$f7YJyP05Enxz=xPDgyiE4^*E3?Z&mc;pS;OsAfT>$N9U z`m39??w?{Ni!_?mspjk>3uOz&0W0!EpWHBD4D>8 zg>_QVMA_|iktydmd`|&xn2NCLFR(k>y-9e@^-?~@JV6C&C+CCe$qYISC>g&Lho+`< z)J+w$PV>m6+j+pq9=LVyy5D+XMO#kcGUrnNlGD(Qxh3`7u_`14Ann{BOKL1o@H}z9 zy}&DM<08NC&K$pllFV`ikYxYIGtTPTM(PGh^}O>3CHL)cD*v z@0D5N!X76-@OF~=L9S=-18?HC=8R=R(4DB@3v7w{ftxC_$~ByZoTC|hfd#eo`>ziZ zV=q^eS_h~tU1FO0u()0}IT~w`%zEw8-}wi-_YgFIW>mT|e};qj>9!ixwWk>(es(2T zW$ER~TR+w#lSjaXAH}}?Ss~~9$vGBZ@X!#(v+MY#8hu9CoZzXQr8FiS0)bZm)I3$u zeo&!t;hgSx35hMo1v^b*ekPu11>}y6wp4UJRdyKid*K~UXUe_yZB8DC9O{b$|B4QW zs>5o4yJc?ukK~Qx_ujEvbIvik$bkq&g&D0ovEU;Ie>;1XNmF%KP#i?(jos_5*b9a?~F3DJHTp(Ax(+MI({ z=tADzYs$X1SpP>~BU1SCg_pyzp1??)Jk_Xqb-)L$`wpN4Hp6L)#GY~xil3N zqtEbUp}?)%q}DhoRCAhh3Yf6RXn&n36w=?WgY6UTCU=$afcq~t4!)&FW7TkRf`>|vrGpz@{O$5wg1-^lf10^Jxyv3Ws~c?FHQH^cZHp{B(R2 z8G0SfzycOZZBm@c2io+w5Em$=Twu!+2-SEF>iN9b6}l8IPrzK}@N)T5Yfs^~ z?Gnkt5yi+&xi_ zlyOQhqt}A?neso;h=*6kW6`FTLLf%YQ4AU`ci^5daj*1<_lGEwNSPR(zB$())Yj)X zcrV^En0Z`TzHT+(xn)hl@|*wJA3zGR&*aLVzJF%(z2j6{TrHN%NnER_Iue|}dpErM zhEXYmCc*ymO=_$>jH*bQ#=hj)qGRo8X$J~Dw6K!%@t!fmHC;00!BB4|n*%8uW2wh@ zu?O^ea%-mrJ0BeYKe)WNoa8n^t<1c*v(>LPoFSsM>{J9E=bd0P-ZLwRUx5WvcN~YR zHAil7L8j#}lL}T*mS7kecMxkv&4I5ltVo6$9>U}2_^5MbIXyl#$eyYkxY{11f5GK% zx;cm)k@v*KT!OG1S?1}@TZsnbeRkhzo$tdSWrG5H{+KSvr{_wr$5{GiUBw-su}gya6o&Uptfv*~Waz`z%;s+<6FRbHDE`l+j&QQ1x2i8EZwL+X)0)bR*Y zD9vNg3<7JjZq8`v0)ebtGliXBHy5Txb_@7H*elvM)Dc1BGNUG$z8ZdVKb~R=TLxlg z)6~js-KSfA1^B|@d9rzgLq;)jULoGcccbohi>hhZ4pT+1z(_tQoA|cvUiEH^sH);f zi`E5wwVXTFS!@Uz$bKwDsxqRYAEUOHKx!ZKx&pIRy;v#dz!tTaDbe(CLh8<7 zUl=p(PutX|GI5(oRwGMWEm}m_sKk-ohAuO7x9h%xx4JLKP?b{Qf6^wBqRNJt0w=!Nj$~gFaH>0`Iz_gtHl($c-9#Dk_g|V{A4QS#lgVkMbhc zPEvdc9-PXvdGiJ7D9Rg9^J;#j-|Sm)W?7qbVGZ|IXY!8_5p52eX<${f%FqMd6G9rA zcvXy(En_`3LrNe{bB?xTVpS>tx)dW2szUh!_oSC9g^Rd=JFd9x{$5Bs73J$u{#|rq zQ`#zvjr2C;f8(%i8gE(AHD5l5OADZQ=n zi6-)2`W#Rp9xJRZZHCEa{#x~MMLlX)Pm~Me-uj%e9vffZ ze=PC2b5iraUXQ1#2ADUmuXuZx(EumILTs}Xx)jVTcwQjmoWgbR#X}|lsD7@c{DT^R z`evb{nE_v-ytgYd?^>RWl{V4k7rvcx-81A(m&ikC;D)VDiK^`%QZ`kCam2I3hMJ3H7%cLDo zHEvd@Z^-p9O^Ws#wf1v8MHkAxs_Y&zvRqhm{F^Xcdsvvp745Eb+j7z7dvs=henmeA z{&ZEAyO@Nu=Uy6jmtsohys$P;HWTI#(T&3K$_Iu~*#%v{+rutSB3Me%Pow@tiy39P<@2QXM}$&vV5(P+GnK)ORc3 z@-#0~TZ5HpKtnjVppiWKO~dExIGx*O4onTdX5R_6qIQ1Rv$>8SkK$uxPnl(TUFL;- zWM;V<0wchvKhb7>UZIDgf&(ZU*PrsT#$ABNQ<0@`t)!^| zqD-|hL$(<`LOhtfM9m&9^bcZq0go5xRkVj`kwp!-5;C^*y4mc6ulyNuJGX|;)ZMO5 zsrTVS=p>b!c6+nq$rR-!*4)jD8#eYu-}j_2c4W^UMf1&~R}n4$^` z^}HbVD2*#&yV^PnwGtO`UD;bIGZ?1JbMEw5vmNZ-O5JW+P^l-rWj~$VdN2mWE+b_2 zS1Bx5a*BZiAkz4tsa z9$v{3(_{AE2}2idm#x*k8#7D(Aba$zK}hS*A&%BZCl9IYVJgO<_hsvtAq;`^*fB881cnOKB4%@E|6{S8!(SIRR!{{KE4S_kUS$a=8O$&sLss zu3Ee2r~aJ*-agG@i!fcuGs^qaw@26b%%Gu)hhV`WL^g zb!L#mv@-ltE4+vJ=iV+M(|Ky5=KS^^H9DF}urf;y{#YP>IP#k1CXC%3)N=Goyu?OWDBH4?Ts?EyAonv2x-?Q8n}7NjMM`K&3QY8@*ljQ&4ItB z)J#ImIY$WqC6>o%iIYdpHLP*8MT%HF(S!12(_f|?o9T``i^uWMGt*?+{*ViO^&_Jv z8VExir@keW__)ZLXo=&VR;juS4O0q~s?+E?4JI5Wwg%kvTg6P29M4&|Nvk-;-kMZ> zz}%@=f6~cQyBqz|5zwRWq$dO7z+4HQBu^xUXAD6TDjVM9EC%!%roWo&b5eb2@ObLQ zFFmoJq<`-VZjxK%2~XSD-0oAHFeEaLG4c38A9&smghd}cpp6cAZ)0>$UTO2R=fKvN zpcY?-EUW*)m0dUN#4iFngOd=|WcR>MA?LZ?hq{|^sw;i;WX5Xu4lRUp#^062Z!^WU z#A1rS?G+b1ypnHv#lAIpG!!)M@+@l-#4F@hnYzsa1Tz@==7W4zmhNW-#@=*iZ5|D1E>{drK0xz9L z+}{O9yn4Ku`h0(&I0EPO$fo0A_8cQt7TGyjp}j4x>ixwpWZ|0t7VV5!{i1U4lj>6; zC1k4gMMa&CRc5y5Vmr4#)4J7al^a%8PJmhV`5Q(x(8Q6AJiXe>$d&M2`QN)MlFJes zWCzNpe~Xa8-qq{pdg02l+?*&c3icshAO(r}7lGxE37>0n4*+WoN^CJPNKK%GoWI$v zSO^bcv%Z$YTH?w2HUsCtHjI!~w^5p$k3sJd_SY30iR74ffLpLwnKA28HRFHMec%Al z80a+7$2Z@@%*g8X|OUX7`C+5Np!QVbL4sFN&R8iUzb9#K zP?~SRTcE$>v22>1;D*_#Hqqx(=@WBt}T0bO`v748Sub}yhE{H9OO zx9ja0bt|gas0=Y^rRVoANM8igfIie$=W_C`KGc*5^cuCQLV}9&B8RHL>1ufgaWT_A zg(fYK8PfRkHR0U!fNBn6M!597JER*4g^E&o6Ol$e<6X8a2t)i(G%CR0I#4LR0M?kZz(`J0O{}jB@3Os$cW`J|StqDM<9^bc?>ogI3 zfV)qX*Md;5DLqIqfLgFo+$>h5RqA@Vvj@t=85~pKR-lDh&y;ft=T7Ch_O}^)p7%5tW=P70Nds_lhMw7DWBk zaQ77(MM%SdR?5Y8RLn=M6!s;vl$+cOqHLkzOP(>mS<;IL=N0m47JD@xSea%C`oei_ zBOxnE6;6+>6ko5LQZ?>i?@`a6R0XkMr$hu{USgEzGKCO(wW!vLcE&SuhNUa=T3TTL z05xO(@h#@Cy#=V36)U2*|4N&NKP`Y|mw@9}xHgCz;WBN(XvjwqA1bSy{RuiDLg{`W z?f9f*9urcrnc4cDb#8>vs(n-hB#`vQiA|1{|ski;UX83_d{f?m7lo>w$qcK!+YqIumr^ zqPj?n@~}pHu#F6_dX|Vvu71yn9ShMh#XjXiahDO-6|@=Rt+iOrdl=Q=z@!0OSO8xg zgc{SgNT4Bmz{5!qpV$?%W#ojHz5q3m^D)H(_1zCMI*v8yM23E!>QRS}8Ltf&?X|f* zf?M!OmMbwnzv5HR)I?`RdlSIuqu{OHOpO?nC3nm47174CRz+j)UaYjo99L|Hz~c{$ zt+J}oM1#R--#nxB5Gc?9OGbhgNt@Aof~}37y#*Z`sf8Q>*2^23@bVd^q3)Nc!t5^V z$qda7JxL=J-jSyQO&nE^UzTv`R_OfU17b0?1Tg%W zh5XvDN&LCaCrg=jAl-yt{B{vxy5{y(05)wpDGe0fA4gElOKs9kV!;5WZS5)WfOFhL z`gBavY{m8|R9Ra(oy1vx^e@&BOmx5yrpf!BUF_3z51{Y(BW0BXN8 zQl3~XK7uV)07|)8>PZ%g3s68aj~@Y=XL4tI$#Yg><25+fi=@ANkk~ySaxg`@3O43I zHC4nnQY_g(A4NBVha%VteS_nE?_XSlkK2T}5DgVVU+7HLJ+)IOuNd?R<|QTAxB5%- zZ*_@unf2Y!Jpokv$vX@NzgYMX=&+YLR6WzhQ@HHEj4%{z8WQD!ox>aKDLOIGl+vD5 zkN3*X;8x0Ov5qRE0(Jeh?OT_vB^jvSirWa?=F!c92}JtCFk?Lh370wZ20vQvy2*6t zP2SMmF@f@y;q01<#85X85`wJnBVgkU4`1^GUnR-cXOq)$>p!01GNlusU5;;_-?p>6 z%z^)!HND>Xe#a)#vqyYQ^`X?}>y2u33I`Ffhja@tUMPR|qvc7;CjtLH&(Ml81FR&W zx+isG+M$LUW$7;q54aV%)u(VB!927%*vKzz9Rw7K$2qX4Or5uUA~!JZEo6h`UV=fgY}tY(w*0Xm-F7v zOSpWKdj5SL5*zWFGYoU^R!EC1nTvSh{YFm}kCFNp5b{qK?}^vC2Jy}u*2WI(vI2C8|SMc5VH1&iQdrW^6p{G=YVWrnP^zXMMcq- z^BTd+yT9boWcyiXHCe;oL~OMf%iWw~ay08y1mgS)BpUgwPFq2*RWtRzh}uR@-1|fR z^XHrJ_-{+miVohL;&G58p}2~6hZ5FMsTss7k@`DX72M*jL{MtQ5IMx*L>Pg~c;a}* zifWE?P9dF1#!jVN9~WR_??t)5YZkuo%TiiZN&(`m%XVM7i5GL+@+8`mOW-H+Hj#PF z5m|zj6f^A!QR^dhVSN+Osri)24k#V|stD;w?0P@P=)m+MrIP%|$?XLTW(01f1Ti$z z`$2kJXKRk>AJzQ%?USc92iE^5TDPg$Q?`Jlx#DnC$3Piq!oXA~6)vD}Rc5ZOBRr0zbUCc9Abiv;hCdf{oqh-G}aN{0Ol*xhM!MsE51u+ z2EQ@~60x4(<_ILF!;Ch$zyJ58=q@)4tX}F3gZiCdC``5}xkzmzP0$8!8>N!X&iGP8 zY4{wl;Gtt@)M-Maz{>O)kwF?s+@47Po43f^t=@NKT)JdS+H+1>32BFNPf`gICTTDf zR_DoWPrXq&@aJvdS%B?!v0!%v5vyP466-gfP$9?-9@(f&)^CLHIV81v+Y&2U!YagV ze;)3?*L__*$K)kLp|Fd5SkbG17NsQLmli48>Q%^24f_KXz_s|T`EJuHy!#W6mvoMA zy%42Ec%bvV^s58I9)(mFCfOXMQRkvXp{Ml8TUfW<(rfb5!2QqJy$VI5lmuTWap18nER2@-1SPx9xK$SgiJZ?uRy@Tdlh&$fINr~y$UR9PEIS1zy}FT70#b}@su z9ac>GauHiUoQjgUnMZPg32LWM`Udrm{e08G^Nw0$PfnX?F#PF6!eufA%+R2a5x8Bg9;0dSd@B*UkOg~_U6^twA~mcA$7uwn_1`xhEcFvQw4kRb2R zE>-;+iIh~bN}<&qP5FgjtRX{t zFHZIe6MTcbN5Py;ZutV|w_mP)h4Z}UvB?e`dt@(D^F~;~%8}ieyDsOpXi%C`h5e@k zY{Y;JoQlLhr(y)=07>$aqITt z{q;-0%_h;Un8IvzHKziM7$7CddM?ANY&MX1OT39yre#wP%ccZNJwF)V=Mh^hkrYp? zxUx;JUU__q3BzDCofk4rxTQD^6k{Se<6P3XWhxXo5Ud) zpSp4m766QBsrQ)C0QJ|4e=^OdnrE|T?(?|Q7&sEoU@xwAtH*CPwPg_S8AETz7>t}C zWYHCW_Bs}L<+6cE3|($|`S|>0SchI}&hl2CdZStBkF@EpT!-w4lOnZGB@Lg}(<=pH z!yHm=)5no+`^n5;Hz5!p+NU2J)8XWRB;HYn@ZxA+WcSGZPp@2*mOq(U*!Z8W$!!tb zI{L&{*|!b7yZwSV9)11!?K{6#FEa2RQ2l>~dYKFVL|t(+YNRf)vf-Dl`UH%7?-pov zIg$x_t()PcZ&q_u_GN8deImiW{JY+JdiNp`?MEt6)?rlLWe;mCc`)H!{>`YAm1V?9 zya27Wd4CyvCQ)}OBhyfp*g#&_cZiG6iWlsnS{5~8`4n4L!*fx&9PtEv9FqLZgeluZ>1U z)w+Gd+8L^7C!vtUXD@kZ`E1I&PvO2j0Hw1c5irVTCBt)%$u7@L;RRJdd{||rOWf4WmE{EZ>nbn0?qI83Tpq{vN^_ou-Sjyvfom;9MV*GMXM8_Fdeb#ooPjg z3FN5bV8s1QG@r%OKtunq3V6GB)0el2Z6sQ+`_8S%>?o2@x~?>W4dmjFW!h~2TJX-X za181tH3urxXDsjK)wC4StzAptT%V@(nd!GN%xTWl{br`rsGBF26KH%@zr4Vy3Lfuo z*6?H8lVR}+3)QbVM}QOAq{Ic;Dw%y@RXM0I^{UEnP9DDE&y(0J@SSBK7~SD#%3||>5q8!AQKtR7$HD>&R8T^tn<1rjMGzReJBAPt zMp|-I6eLG_r~&CArKMG3WGLwd5rz(F7+^T}xV!Ip&)NN*^RN4d?mW+P-`}{d&jkPl z1(Vd%k)WKX$MOt#^J?EzsO80}C)U%mKMvhcgTghN%WQPr_OU(^YQfIAe#4i&l;hd^ zohDo}lR4f7l{{cX*e@$^yrt52_8XHBbjD6{W^bgO6vY1!(3AH_G+QodyC?~PZ3+e= zDn1MTf!bW2Rj?ZAg+<$td?2$uOGnU<*r4xY?6U`CDo!QKNTx0CZ!68C78b|)Dp#)#tAmF7nyRS?*1X_rxuF4)p2LR z2AIzg?c-<$4-fpfY22e5BBXhkW--6IwA-KN^LG#aE<9dHqPZeBerDBdOdGs~6y14_ zRj?W>#g1`Z(;tY9uRnhC!&s2g{v(6tQb8!jnYqFFp~a5ygeICb{QB-Q^M|Xdub=C>@H2d*y?%Fz>Bn;9u>lPq*Fe}$hc}a#W3ojd+Hq2oEs;xrZ9p=u zfZ{2v3bZLtw>3R41Kj;|)$#?g7i)JMA5QCd1(~kGC*B-2k`9Eg*>k>{ zmyGqmL_2Ze{@S0^7D{09eEsduW_@7yNzczuDQq{JR+RMl)oHe$;A-W2I=q>!=0Q=! z+8$yzURLy_tdP8ky`XsuR6m?H$ZlbKd(Om$bw?4n@`Xgr84FsZvW;z9b_wtsRFt`1 zXizcV<(}F#!n9=tOWWipe3_-L8S=GXqn5BWU{xI!ty4x&t+A4NKZ3-H zNbWXU(KU=Wt`ZoXzsh1LUi;PXZ+N!}&BJMg=pA_6{V2>uGV&OhBGx0I?~&Y}M(iVv zl6D0v*gf8SBJ{|>E>c2F>r+DJ*uA#m(p36Q>y-P8CYqMJ6sFB5w>4ri-9b$SjgIwU zRqUlZ*Jy9u&UGffQ6BfV^#2VusH)O0oyp**6^JlQ) zKbZeX@*qD0-d-3_{3a-hcJ$WV)7lFZk0P?~ty~7@Jnx!1Dux(v=33?{Sb%Zja*c|5 zyG|!fI-$J8wS4yx(z9({Z)m%XN#?QmlTh4#zv`^IUFE%)@I4}X1Lu(_T8gQ0q90%j zNktcUg%7LYY#=q^>*v1^7m-Oa3u_uTphjP=o1wI9uczk)XBL$~qrr`>YR}&sYF}GH z-sbFQ|B*+|c0}8GAeZ(lx94u8!t!b$Td~B=W#Tt{Oed#6TBI zIG(AfM-E{GQk6$-XuHp9hPKJHY(9l(8oR1`?%D7;C5f&`DzGZDSf!PzVjLCX*=K5Jzf(0>c@spS6ASiO^ak#eZPESFu{dp!w;byZ~7) zJzb2L8r`r1zY?eC=KO+5lSoXs1s6m0fIWIkR4x2h3pb((8k zD*(!tJ3de4aTW1Vj&>f{hU5L^*83y3Uk9Z9{dDF3ho@_mHr8*_0=&3-#(c<4-l51H zQFL&8o-=2Ubk}HM{Mlz&FU?rVO7DJ+#ZoVQ;JsQJH+>;TsfE2rR8B%s-Pj8n+qrSnO-5c#det6}R!mHPT$OO3SzBA$hiW^5DF$g09KZ{f*F+M7P!d z;7od6$|bYQkpXgn99OfYP9Ox9E0)7c=uZKGAC&qzp*xPyck#8K%hgbX-FoTJ#6w(N=_U`y_ zX7yyz;Q92|m3H0_9s;vQfh!yq;!?ns$!IDLgf)waiw(ZI%9@jaY{8n2VbHA6=uw|1 ziI{e){S(=3Yq!PpP*SgXIQI0%UyogtiR`^uF1rM7?Rk1`$T}B@gKJoRf`s3fuoqww z%0#e6GbBl$s^aGizFifGGCXz2ngIXXwSfrD~%p zE{k0bF3Utacn}%IqVEFG4bZn}mB+ z0>8MYGIKmiA?fYM1bWw8$O;n6tRssfS09Hq@D5n=ayg726to?3;=t&@O098+ktWOCK9}NL5y2K)8J$!C`3-1`>_`A?#F}v zio3>rS^L>um@XL5u1u$tn4MgMA5KI!Q^DUAgdU|O)t@+9qp`y%%}^(zb}7J9Bm3j8 z+y>ruhWa})g0uMo8l0PGfnN&&(&CEVB>RPE5FpxM^=?~K2LVpeAxy^zsb1CJ<~JoQ zHKP2UbKs^Xi!r0I4<(nW1ZOI|uTU|MCWzt#7()ziS1&^Z)O%<;-44Pxfus_(bR48i zV^0rao$kSj{E)NuI!B9#Xu{;oH`@@qG*6(7-nulq?lTqb?F_Z?`q~4I?d)f(T0kF~ zK6T-B>io68Z*KPgsb1?7P+wiuYjofJD%*-k;_I4>C3;k8uVy>vjo)ly+{7~;#$Xy~Q35lSWIBw`YOlrd0{;5s;HG zoM>28ftzmosihoQNUp>o392ccV50PM)4wi>SoIEhyFVG_StF!Qa0lL%I?xLH*4oU3 z-h`nm9R{Y+xS@vky&^eV%}}u+xzA5hE2nQL%=R9h4LJVvSy z$L<%JVjwzAdm{H>TsDW|HSV-+@IuR1Ge}f-q`Rw8>7wp4g)d!otWP2Z+1jCnJI?k;v(a8b;1stRT~F* z$iAe9jMl_N5#cowQ^fGCJsXL!_2fBgLkZ2FDEPTx3&03_cx6Vf75V({WfiF-`r4WGiTEB6HhhtYa=-r`m2S10y6_Sab#Ly;{#V=1sn=+4mfZv#kd_X5N!O{%?ue zZ?c#AF~hpi&fA&8bx{^jmMNR?e^3(rwE%R+LM!JMh7;^oa=s0=rY==+WEE6$A`$Hs zExP(vfGzX&347@K5MibaA-P7# z_CwQ%5XC4hv1?M71T!2TSBXPy1?lhT{W#gvVnOvyf*8hOYi3zpH%$rgJ{-+ zWx3%zWA?N7`Bk-yjKdkF5Gk31sn^R>NW2ZDL_*5KXhfv!?m*!?$g|-z`=MC@dp1S$ zOGaygXE_`1#Glu0R~+pKtPANrB4J@Rng?GfzsH?Z(2atWWP@Ud)=*>5MQ3NIC|{)R zXqT4Lw`(ROeiK7Hh!b8>RPdbloFx@uitE$IVI2*-*%o6rEsL z)8|N$Cm)oTmfxhfW?|uus>n&Ovsz$nu$8xuN#um%E7B27qL>`SW?)E#y5&=r*=Jam0 z`pXR5+fdtWZ)NgByCSYJz}9{80e2>3e=sxbOha6ayU90BRqtU<^p5F2D2h~A&o!_j zl%*kE-yLc%kQv;eN7fs!_ncbaI;mD|uqCPBkV~wM%(VN_L6JQfTQR*&>mz0064x+Q z0`9edQu1E)A15H%HwTk`S{BeSZO&W7XM2Wo19cZ+9JUdgTq^*tqmm*>oK)<{?Y)7@cZb$J7tn+-^iDy@^ zv}C5TN)7rhK#^eKP6Zb5Tg~5GX+HtgNoZ(bGm;R#H4qj7I=8G6t~evE@QsQQa0k^f zN@3^iyj1KZ4`Ev2tF{zi{2dBs!zIG^7!o1-#~_*cQo6Css0g*O$3~QBRuWt$@~gom zRySc#`kyBwYr>ShNhzzDW?X@)-dC(qw~$vn7yQS6zV&DS{;lHxUyL+Piw;g^ZxrjS zF&kYz#ucg{I1t`)pBZ-+CTM!yP1CsIPk$AU?4>DPnUVF(@r!50I7?yk>GQM<`rsoR zxEq4Eg!k$k{eH!kHoxKQF>ub+gs;Dl)cs~I3^w)!poc^mCn}K!(Z5G%f?5+y#pt1i zCnoo&Y+Bz^Slo7&Zc*q`r&)H%IIVl=F8 z_q>1DBP7C{je_pWnwUI4?mzYh(iFoR)|dIXzTG0Km{_;B*6uf|%yl8F9@H=aV~g;p zmNPw*X1e1&A=-=Et~q+CuX?PFqVe-I>)J~4C4uFzy0;JwLHQAwm-N**NRKm-`*yHT zm&KZ4YJ}h~S}@tgZBvg{*oN{0?J1GTJw!;|*_Xk4^s<5AYW;@Vs=w~h-6dk4b;CyB zQ0;6>hNeScl)ffRL$?jgPEU}%G-K$x?yx(yGnl+O#OQ9>)CG#VA_os_WhofR?2cjb z?RTi%fkw&wR*lvl?_K_LAe8+3fdCvu<;pAvw?=R%HKiAi*XQ-Y8?sY<;irm|@f8WC z8%^gmtq8C(mvkX@oza0O^CXwO3n zR~h?$ilV#GBg~!$k0A$h_X^+>DDLc**BOf?2ptXlJSQ|7RKszOXg?=eu_>#*M3Ml^S~__cD=Ek(vR`KvWO4tnWlodwZqm2s4OVc2i{xCy zCbgrMTo~wLW8#GK#bxaGxc1`#$Ij4a`vZ`WC68A)LDd@XjPE`Toyjd}&acj|1?Q;p zJtjO%Nv@>Xvure@OA-?rfn&E7RSQPN+6;raUIG@uvdF5AXwAnEvVQLPp)6ecVLLX1 zo9YHAcBP`4=acyhP7AuJWejbNpqgRj!8qp814< zF;w5M*HE*;rK;+Y=OAf}?DTU>okI_U-k*hbUGEqNn z&Mk$|_L${*+q#~)Qnth}W3|4Z*=jx+rtd6$l7&@9ZF{wnz#mu1>X8T)2V);KOGQ8E zzjq93U`s`BPqQLN3n-4p?w*csz2)4kMjo1{MbDnzKI?ne>SRt25R?gAp2zRieM@g% zt`QIR$>pU@&%+dcwD=9_T?W%uTplYu(?1H_~dK;>+$gyuul(W7bTwuSHQ8f37Oy!U|mR*!o$Zu4pN8SiRn> zV&fR^`LF=VTpS+k)@#5PX3y17IWWLr@n!L68mwuKy6R0tWN3t)s6wQG(k&PcaOMvc z<*m9VU6GZScl=H_l^s*0CCY1?dK2G`5QzQqusysXS8BF}vEp!ffDz$reAtB! zPx0RDu^lcBiju4(&QS&Ggcreir5+xuXBA>T)qD^ZKyF?lk3WlG=J8@Djxt?lsMsnr#axZ{^tK*9mH9`XXnC-oWR3|0Rt2w$pHue3 zjb|RVM}D!W(g~%A2HX~dzM_#|Yx3_=1@J!IUZ-&{u^3fg7c22u$r(tqV!GKfPUJh@ zUNSxIZ5Z=fk|7?A(H;}SaJV5p+=qtzs&3)(g$EjJ^dP+EJ}pqrk8G*F5!# znqp0idq?~q3rv=^<$luK@qQwyucA{tN_~)<>nY4$7-~6Y544UV7qYoUDH0=m%6qT zbmXP@@DTB7gO$k10ss1D=Ic({C3Pl#w4u3w~cB7;suR zONll=<;f~5;$?{Pd^-1S7|B*#!;mo3P{SP0cGH$*_kg?&26Ro7J^?8E)M+6p)ZkG33De@=Qy zqWQiL4T|2JH1aOlYbx9!DV_S904>&)!xS+d)_+^Bp`HYkqUlri{{nrZ!~N^2{Ya2T zvz9TlH?&B#xgx&`9h1M8dEdk+o;AiCzz7yuaV&3TZ1NarTScf%v7DB5I|OA&e^xL? zlG(fJxy8@w^?<1UCAk_duH#LCfqk zmcf~@iPk%#Q{nImckbj*KQ=>Kb#a)pS7mn-*VFlGV}DV2BNDnZP65r&n@^O=!l&n5 z@1;x;aA+HZ85^`G%J<wJPMP?U+Tlxj4PR~Yfi%hraP z!j&P_rLoJ;;1;~bVu;wVA|fM8DH@othgP-@=pi}#f@o98aclX-z>D}SSYl2YjK(& z(<)%x>;)$e@uP2)Ra#%iD+}!7-p)-yVKU40H> zV{Al4V{1@BU?kX#VIY_g5|5XR=8pLIdZKw+sEWSJt;wN?*pI(l4VGorAGho=W`Ft7 zdo(h0`xDxHjqR55Uw0Al*4ckS$D65LVa16Es2ARKVfYU_bwxys$>Xr0<^pUD%8622 zz@_{O;Nn|-0S571Q5zODfG*``6O(-*Dt+NH@t(thf5SNYl`GS3Jh?S-1&8T@F)N0j z2O8@1a%Q^TKEt~(ROC#V6#Fybv#U?J=U))CUbfd3=Nb}A1Xj7&r|^~A7l*4WZ^R%2=MU(xi%6Y`@tY?fk<@oDA7lDnPHT&UvvN;XQp*(?OcR{ z#cr#o6^8|zlDf&Zzp3^Sc;2!ymeA;O{0DFEDl-UyBGPb$n1pDF7cF`^A&UP<{(u!1 z5UbH9J8U#&+^a=F#q#{Lp`zn{;>LD5=eKiDvL0AnT3+fNh{mE-n_fuA3~VkytlngZ zSLV|;58O1yD(W2mf#M8_J^o`^OQEGTta-3^eRx7-{?}i>2sz^{4haB*U z@a-hFyTlK%Fvzn0xun+I7`*?lOX@$Adl#>k12{JDw8X2~FDT*9GS=}_-Ep+g@LRs* zH{R8s42FO}%Tb(Bs~2m;|H=xqFaQwN%(%|u^16%q$GCz<^IqMMf`%a3_a(*XEZ7%a z;CpcJLYyBZF2I1p8%6jd;2|h9Bi=deKjkpo$38R9(84coV}!Q3_^mTBGWR+%-9H%- zM$ZltJQyrSFBMq=M^+ZB!d$N04lheeVp5Hr^UkvM5O*9c!FI}HYba~?umf@zKpQ&SDDq07rYB2McLd}{h{Mxsc!$JtPdSt2CD7&!(O+b}&fIOYZ=PZtPY}P=o1iOPX&v zTL_<|ryUOn?D7A59YZ<6L>y}nT!Dik2e&`>lDD|G)F|#~K!-uFaNPiUh&_Ye2?0#W z?m3DEXD}hov`AyL<&Hws$CLJ}h+e5b;uCFf$M=77<$(dyg0(7W0-m^FoT;hy%c_Ca zvgs4Z5{2{SBy2N@Wyk@D{7)BcO$Cof_O#IG&m2Z3q;ME*P`l*ZCVwmTUEmyf%kusU z2LIykGUJIfaZ;>{l`Gr}f^q?cMP0tr7ig!gzP-S$A$5uvrSNZq*G*$!b~BbR4U67a z%mB)@G-XNZug%C_1@%;kOW9wm;rHOAW$RiO^Qj~COA!z!XOEUWc-z96ODBG>O>7kZ zr;8^wnIs9@ks7YjS^l&)@1jq?D=CoKp=}n=qF{Gc(JD0qAcN9CQ2C+xr1b@US~A!4 z2gGoSip6H53Sgx(*V3AOY@QlD`&h9zrU!&`Jj2@L4NddS1&woIcq0Mgo;wvDJ-cq@ zEujB+I^6=RppMXlz}p1WH4>v)g?}9wJ7j`J_9}@`sPoCi-Rx*1UJHuoe2J30<@4K} z!?FUUmD#mk+nX#lv+$Umjh_>#Kj8NU@o)lBVA6E&5^4WJzIGK6t7YjU43a2rnqCA} zF&0y*qX9>a2hhSwFKE_E@k`;MD?sR;ub8LWGJi`=_Y>riv6r|Ox0Z42g3uB7Ag3xm z>Al35kWf*5W+7(g^pQL@u?qN=f3PbmA$G!oDbr)M^Kbi(OlIUz_!A)q-2*_qUYD<3 zxC_yiS-FS7=KIt;16k}|4CU4IJi939`+F%2QTQi2b7k0>&SYc!lHAHLVQ+p7;y#Rb zeiHUBnfTT1PLFTe718KfW`W|1P<}PklRv$Nv<(A!a=IpI*G!W$|Xr8wV9S#Rm zv|gAbK#SRRu>oU877%{IptppMWNz+n>|5xJP^Z`4V})kh%1hz};Y#P7qD4PIqg|6g zT=(ao=%PB}90Ty$9Z9df1)%=~*!msvOc0n>CqC`rnzMzr5A>s?O*`;YYBcfcvF-??h)!KQ@Ure$yaoIN9`W+0&fR5z|= zEJqO2BXd*QGxMB1i^!wu6AxzZc1Mh50)xBw$bxzTbt^L_kv7IjeVLr2BaB(xpk}%J z3^&4?yr|cV|NCek3`naiKk9A&amoAV{50TTy^0)R#+2b6VMBTgqXEVZc5!JIY;A1r zoyvUS?j0JMWB`h}+RMP73fxm~JT(?>P6JY$fLgHFw-A_zj8G8IG4Rj++?xiF$hY93 z-T3RFy~^NKJ#uz`yy)*E1=ibxF6GkX#$Ef>3t`;`yc@I@hc2&zgM?Eok?U^ync7Dg zFKHTfXnF@;RNsQ7`86WABRqYI87NXMQ}dr3Bw+$DSCV6UQc z>kW^?0u|h@gXY*_Ht6W0Sj^`yBpcV28EC8N^uPD_mTCFopDZRHQEq}el1)IjzUu*o z!iQBeb5NGUL%rvCs0d1Nf}G57c`AGLrnPL1Q@5kF&~3U9eXHs8XX;0FEd+2)Y>PY& zHEs2&BCa$W91&|wMnvTlEIFqPg{=Nu!(F%1CvFyapn~dw2q9n6R7!Na@D%^0Z~$Uu69BAYP7u@L8-}hVU%OC;g#}X>&M5(vQDS~garxPW{&fANRQL&a~HXn0s{+M5V zdh-8sDh|~mn&06YQX2SHx|_%VE|Ps4<(LaLB0xzuz_?dci}-bu{H0J4=F83LeXVLv zE&P4(^7Y!@WqwzDVS(`((~A zN>~@r!7hma9Ry80tL?4=mo@&wlFG<=m&uc{*#kAQ`%P(Uw8k27O^yUG?!u zatIoAKq>YHs2Es|$DGqX6og8*xek`cf|s4{T#e6Cg8H!@2x2?SH8}YT(h!_px6<-Q zHF?N_T;p$1Uly(#Xz(Ds-ua5F_ZDtOS!n~iht>$X$a#D|>bkvr%&!W+CQeub``IFS z6%kez5&GwD74!<$1%V_m$;(%#S#jT}Ivx)$b1ZKTN71Gvc8!d!tNwZ+CqL439MMaj zM)6y3oy4(kPxmj`v@aii(ToAw+vY;iXm!BMc|^;;uxJwTO+d7ZV(0|!g%*qMgM7)b z0za`X0S?Y-i_)%)c|pMhSlR6v5Z5pa>zdo6y%c)%!`58yd;0~X6=dTt-5vy#UED{7L&QE@(7CCAv%y0K;irRW7h4eS(=ELZyAp zS9ni-e(;hs%KagAEh!M4bktD7rOD>Wsd?Nqh;#8A%|A}PGoT~mpDqA`A?=U)2hCyz z2F@|<(u3U@3r_jpQ2bUrD z=GeM^R*NCH0t@ADCFo9SfU($V-vAI3u>gz~puW8y%ab{H{4oYze;(g>*uLh}3AXxL=bc7b=DbCRPhGQrAOhv_QdJ`bW zJb_%#zR3=c%`ZM9E%Pq30agPTet~sv1XF^4uaS>*H4sx{#mCk`3MzI^xKWaJ`~P=T zdQi7aqH4{4eu@4=9A~)lU;*)_a{PgUGjMv8RAiOZI>d53IQ)>JM0*kpJ^5HEPh@b4 zauGD8^Ri_YMdRMl-(9$T_!D_(BACu$ov8rst7*R3chc_&t!1hGYd528uNxBDPvtNE zxB#d^?RpJXsarB8M#%1Pl|-2rvdglV+x%Kb?*4D;Oqn_3Ksdr}v4P|J@S30=03*kY zYF{yoEMs06b-s0i#`UkM)*sWuQG3@_twv<)Riu**V3P1! zI>i&q>?>iziyDv!)y#5~=;gR?>Db(SA5K<^Ozl;{;p?lSQr9sq3`3|xkGSGr$5dy4 z@ghMH^sR?3AcXhxdUWgrA#70@=4Vpur)2^zI6iGh8ejKO)|4;rS1aKRjYY6ph~iSQ zLj_IzRc#Q#du4im%b7C;s7~p%p?FC17HRRq$UW&kp@R>&UA@1F_>4`nbXmx8x^5+U z2p5CarL8=7wpjXL36v|+^NV(NB=;*B`2`d^le{H*`8iwC>JucQXq^7|$3ocpPd&{>scC=F>cnW162_PJlQMp9_zseLaGH)a+@sA z&Ifpew2!{hVMfa6r+6$5-Qb~dqkEl=X$}p9yn>x&hhV>93-pFiF;%8xS|}|HXo~vFBnjWEA3mouE>sKAwwlE}UabU@gNUcHMSu6Xb^HCH zMG3F^fXVTt_2F5`VZVc$-FbahXed( zX>>+z&fUf1Ha5!EU~e`!GDg*_Y885Xc2;5k)>1RV=wv78ZvJ2W#OEp|#GwrHoT}b5 zSVSy`izL$Mb7~Z?9_5USG<1 z)zbywRy)QV3ZpWBxvfAN zkA7>)#R9J*@c8ix^3FXjOOWchjbD3z)j#Yp0DnXR{DszZJy@VWJ_%tAQ#dnCHKTQs zfByG|;|20fV4!FH^U2v7B2F{L-9^R90Ak8#ieH*z^Wl+dDZ=iFdubVY8ZxtM zlsO2X3-If99uh9VIJICveKg?1kIBPdA)%j0O^MijaY51qz=l&o)LTFm+W^MU{58T3 zJ=!a!KO2~`$tmvsDUtmjR}pj|nG2zWLwXBv@KqSnZXdRLRiH@bsdzwNv!dvOzOlRn z7;6kvwpXJ(_w0wSJ660Hlr*;le>MUT^}LZzJpyV5w&vC|MN#ZJ#RxG&H3{9kNjuFx z_>#YrT#aObr|*ZM^P2stx>l(a*X^J3!1@DqOYq(To>Ii^s4bDl4O^(#z*UlVbtq5o zzmeldEv6}_JSRJ2c(bp)e0cWH;cQo!w~_fD9vQQBHb8U zBLi_QsA{4`S`tTArZ!a7WxY#q^U&hq(gnU)V>}uw!;~?jU4@t$o+3_+vCaB5POooy z28w~w3|#b^BFF9iB*VF%&#y0(rqabSwmYpY8C|2{%%ua3t;P=Wr0CO^uTHY!UR__g zKP=_Jxgzda4QL&Kmv)=ZWcN(NZJpTslvK{sy#D9a_~_Lf+B=-bpzPf&=Yw>4T zG8&5cIl-YW);f2N^E>sQ`c_U}aR4#YM3a+dYEMk~i48V0;fP_!fnMOepOvb?2>3P& z#RB!V?Ir502Z`$H%}{deaLBHEQrD~!AySj8+$Dh9IArFpp4yjGA^uL-#l<}WhA>v_2(2OqJDaKy2?V~ zaf_~$Fo)@8?vt7|IOr=-XwY?Y=^0P8?74Y@Q2KqCY4r3dpmGVzlhPlPt$7Ln)Z>%| z026bPtX2q-w9!Eorr_CeHj1~5Q30?YND&ZZr_Ms7<4jyFcWtHX2O`GTSLhek*8u$3 zM(!}o`PSzvXa8}YA+2B2HvJJ4YEMSNyd`Ku(ATQg3TRPV`RPmcV2!_^=&Z?ct-By4N$;p zg7QIS#m@9miVf)n+*=0-(qoU!FBvQafkq(Ex^G^5vX4anA)}z?S2r<;cokU)Fo1I- z{Bs;E&txgS)QY;H*ll;GbnV5tbR)~ut{{at3-q$gMb;B!U$JHN z8@pG@{DCiyg#TEwma5Sg&v%bZEYD^zj;aeRX zQ)}?~LJDd{H?B_EyIorPo({U@T4_rOAVQztWDWv(LL?+~WQ(DKsv#@xK1ojTzkjO) zrpKmvww!3gNdqr0kF)XYO4a)|^rTMG!~m&s92fgYA<^J6i$^7cdvVm+NO91mSbd92 zzm*TXLr=(9aY}5rG{viHlHF6<-%Wl`00Atsp2d@v43~-YlzS)sF-n(b0%WrYpc z3;<<@+dh}_B!97wdRmn8EA?C*B|t_8%2C>MdYw9wbLLgrwUH5|Hnm-0=sqpB0s;to+8rbRW_I?$*^)B)yaW?m3#L?q&H$ zI>t)+4}A?6^hbgs5wTTjqgsdWd}ztYcm`zc&7&?5%Ylga$!RyuylG<0?1 zL%3{~KsAN}to+{(%{6TaLW@J*F3f%IWBspRl^v9SoxL4imZ&mNk^G6o@S=QG{qXyX zCXr^#gbWb6Dp&J|bnSsOfW18?f#ROc6=R1xr}F?nR=1cwrQlkz8=UtV77+r%*f7eu zN+Ch%O)HS7FA+UPA&YBP%|$b%!KaWV4k^;CBSm7x23zK%5!fLToE9ma$U>1~SxaqI zJ_lmiRO}iVqHG&P@b;EzwSTMn0Mv$Lz6gz+5bK_MFhPReUHEHI&HsJ*{^5`eAh$oW zd&(mP7!h(3&no+p*wI|3c@!i?#&e-hUKpKNHnQ9+>K8gPpjL1)_-w$Il_kFN{#C5<{&1&lCEfQf_1xV zf;Dp+H7^547XEi=nSz97WRSqH2pjG}yb*i2iHs-{T=M^XeQ&8y8*5J*Y}h%FRxpMf zz?XCMF$-h)8oa;!O}1h`2y?Oi3b$1lo`@I4vkxM6x|5IPt%K}7g5ZaHCs6DMY_~+g z9KPivlN9kv!>XeTwrd62aI`ja<jX@LWe)Xd=DNY)wD8q*lm`0DOaXimw) z(luCi!zK9z#rBZH$fDwf<%V}ud^a^!z@rA5c-jKg zm6%dMUz;$AgIIGZ>3UDrLJH{W&;_(i+R_cxPtpJyj!V}ebDJgz`oW!)kEM+D5uh)H)wG$k0}NKd`fSF0Y4?{6J!+TDs&3_ z$gCp;ZE*AEZ>tF`?Lk30EZou6n5k4&N-ypNSp*q0y6z?|dspi?+y2oeXuro0KP`xL}#) zif^u4j%<-+u^P~vmGTaa2KioqgmytNm7h5>_tQ6_C)SVRE1+*)L9Wg%pe-hxjV2_` z2OisA&D#Mi>5)y0t78mbN^m!19OUxITa7SKIT;@9h;S$eK!2xj#Q<@8%eZA~^ty}B zDoG^$pKF2L1mxJ1C$oiQ_oKO`)dw|t>T1B|8_pr6KB*xvI{jsioGd_`hbL%KoY=b6 zKI^Ygup%F+e=tQ2;vxH6diBT0F{ZJ@X$*f~xWC*Ikm@8IFxYp!LJn)LtOm6@tSTIL z0v1CxoeCM@^uL^9Mv7FKb&dkeJ7;PlinGVhw2sMnLc8RO-Yy2V3Ju@hChu4Erl(%Q zqXva1ga8}4aIOufTx2cTQ>8eCSG!5hhVk&OIIIoOcQ(uR8ZLBLv6D4xBvqLaGi zfz>@AJYsFULwi-b-9}^!8B}cC}n?0W53r&mc#8 z&idh&NDL`t$^HNCd3a7)J^XwrdXY$naFA}la!2DYrNfC+&Ee^J$2GH->xBRvRgo7) zlFEz59Xfir495suBs1d&ME}*Q1DS?ZfVg)@I-BnEbbX7^{T6vN`^|fOQjFlFA=mpV z1DSZax*DL#OJn_8=MrxlZ)0VWwqW)Al=1Yne;8yXXcBJ zr3e~N4ZyQSyRJZ|D#k}E$bb3@4UyWUW$;mam5rA$L9xfEBQx3~y^JzR1wp(aSUs+8 z<=j54ob?cdXVE84fNVDtlTo9diacM+(Q)7~O$b2_3SaNA zoi~xTbVr@KqoH4738bfbCu? zXawLf)(qaSOm*Gc6MUrTM|lD-zYIC~FW<~*L2hG;Q%KBeeV={NBheBLf<=HpdLT5>C=(~*?r^tR;u-rj_+H@26y2OFD(ObIUWBj;~wj!$up z6|dOemg;A;-R6v+==w4bH>~!utCVrr7P3|$bqjOkW@`K(NTJl^BaQL+pTq(ams9~x z0`0{Zj^a2Vs;T~T=pLp8@vcGx`}rc9IzyfEbwPYDtc9Qr<&xh$;kHohpfet%4?6{o zX2R-YJ@6)Y2E)<6%}#!^;`)blp|?lFiNWO&@6ba0+ARiv196QYd7&`EnAZA zIC*G@NSjVg85n%L*4d?l9~8Mxp9!n=`13&TvR9>s+uyG^JYWcrxqpfuCAsze_^{~p ze&h7U++}ZG8JGKpsE!!HKc1dwV7uryVY-c3ectc3g))H^qsZ;=km#K z_pNLpD7}tAJbhc@xGn6xH&=Tz8AuMJlW4am&30D+>o@uY zxuPlP1^#nI1DRhyEff4JAINx#WyWK5Sm&_XEH_oI{4CDRkOWAmD~7EwWY;%(8HjBw zkiggxnmU2UEJsY;NDs6y`k)-NPff2K9pp#*dCaY+>BUo$@js!yel(-BAFmt6`z_d$ z2iRh{Z^kp}KaAy9D5>3#-S8Y?;QAI?_1TTpgZNP!?J2X*rf;Q2Q?tAQgE#vf$xeqh zz`^gIT@a?=kvx>LW!ZZ|*BjLD7cEnng&AGT>5nZ5S&awThdF$+(CoiU_yxXOXCXIM z_o94^Vtvrk>&M-lQHM9T-jDFmQgHwCXf3;n`%G2H!9kNH?(IMR&2kY*^!g|%ZmF0i zIVG9cdb;26H7(}pnE%y$sA@p2nu|i3jY$7PCruO0c9VL(u0c~TySBdF2@5_r!m(Fx zSJt$JQ;*gg;R^M=7?YxAMV@Z*RoX@hpFOyJs3>LbMIIcVDncNXQs*PSjNE%2vtY}e z6m08@@4EOoH5I_!tfgW^nnBKzRhm#DB_$;4ssL(1y^D%D3+D;jAvgf#OSKm1kj>%c z$y)6B+d^+mg4AV}1y(|MNa%G;1~htL=DtXFF=I;qT?#NWH3a2T7<`;7Av@4kCSYk#_(3UvVp zoe+E2q5vj)5a1ko%p-xkZlz1dy%6`1itIaLW3y<#rgk5qR8@VkgnKhXK#8-_G6yvl zv0*lxYdQ|}ouS8e($QNo9gHsP_JMst{=J3bPM^2)viuc9%ji_l-Ok^&hel9a2cXhn zOSok`XLy;dw4#wCf4Z02-5H^=39$YHz67TY?NH;S@#o3p16E`Yn8~Oz%Qt7LhBb%Y zR%vn^sRv4!KTPztfZ7l;8XCLD&F%n=%0A!HJ24jQt=H~d1pu?VVCPo z((%zZM+@3LtM$7abG1C5;A9nQ1R|Z(Xu)9Yspt*z5J2<9<>d4*D<)b*>t^v=A11EX-f>mF1x*@O3%v;`PnNu-LyX5h&?T+PTr~D{OVn$OJ*D9wAuGB zBp${uh9F04>ZYT(ygQC;riop>Udla#j`l}8rd`_HneT~FxINLVx$qi&9h*0Zn7ywI z6?4JW109I+JcsFq!OEENvF{ohYG6J379efAmJAE%Ny;`o5P=pYfs`S>Xr9VE?rbnN zpJN~+k00A_JeKhAYiHaLKE)Cg8dYd!q(pQ~_Ir)8q{Q*t^LXK;r#Cwe&or zm+{dRfD%M47(=o-|OM-ld;Mv{^@_xZZeqAVCn48luhX)cIu@fm1 zhtG!E5*F81#wcdGUQ3z6N6t5A=2psVS)-L*u3XwFUR$uSy)!q`N;=o`!MUy(2lhg` zj>`!dwd+4G6Bm&Oit4LTJ6-~>Qd%>TW;>sHqIvR7k0)8dyq@NAkue@}!7epnT%Em+}j z{LIS8SE>6GDTvDMlY3=c_~^KXcp;$p+(ULed&-flIcQMw@k4>Di?P!BZMTkQn?=KK zlKQ{fIX2aTOnP?Ys-${Z1_(_=d|N#M-iVW`?Z<#4FzcXaigaAx1%)WO917j~&Kpt>Jp-8K&#i3$9w8$$LHf9$r77X%t( z6E0VyB|SgKfOss-@kbj#k+R$kVE$bOd{z@ukMu0D@1zEHr1{0?Y_-51UG$VRzj5!1 zZisC>`T6C^f6f~PSvSKKjvyHkU+>bn^ks_N-tKqJJU2)CN4KG1tgWP1Z&n?5&?3cI z&C&nG*IUOm-M;PPcw49_C?X+UF$L<3MY>$-Xc>8OsCGaw8ZynXNo+(gBu4g!he; z(3!|X)yT+c`!1?C4Fb=FBs+`xJ(S|lYKH)Mv>6?~Ms371C0 z_51Xlkj?Etj<*ods}2BO@>5-G;X=P%zw50G12C2D(3f}Q3%!AxQ_l$XLsR>OEWwB@+t$*(dw`|nhBNa4E*JFL5Z^vVkY-&4J2 zD+pA>E?GPye7V%5Pwm{6a@tv zS$=ozadG@WdU+w?>3jT4^vJdKgLOm!z1*ETZ$hs_Mvj4Gc%9K_6S!*xm$GXMXbuld zTit{Z+@Q4Ppz%inf-sA7_N%{m*{8$!mDfYa<%zNsCs_V;m*)JGq5x2!2Fv&^+2kl7 zFAcYGe}^Ov@PtXxyXdg!e>dpUa!RSMnv#BCGw#-tZ8^L~Mh+GPa5M%M=?AJXyy;BcEf`1(fek+)Yb;aT^`0Jl_92H+*q1v{2F z0mhq|R8iY;jevAxQ>iWgmhwq`xxp|tgo3r?(oX60ReFqvsXpiBNT!6#ky~qh03e_s zY0KYO$hgrr@()k`E{rqqaNtz*M7;K-}{plkn+Ry?ahzp^HW zpJ{q=?0gH2nW%Wg8{-bS>L;?eV7t^MMa+^jjrEd;kUhteou5#}T@T&P3_-ZxP}uxz z1?SSXPn+(+o7mFWzlq%G9jX#`%vo5b-jwoN2$ysyTr24s8aujK8@uDZs&x?Yf#HXW z(}$9`rWz`jxzBK(*mE>uch49D;^&)dah=#MT|Vh`&E5xO~KI?tO`biiAcdA~qR zE&WzWIs!lT9qqwrE(M8ORJ1ohx&jv!w+99_&!{xigf1-+hXEajk1ux#z^b0 zKt~Hp&mMjd*^f8Z{K6LvHY!dxrWX~YnCOuKe@5=CFqoSBT@kA^210DVDTDwqc?p}? z49@o2%9(8u{!K<5_!9N>#lL9I|0&tZ@Pq0PYsIU)k?2(+xrN1wmU}rskQ#v%E1&Gr zMgWFZlJkS7IhDPlu`hJ3=<;lM;8p01O4JztSdSjl(?0zjgZ)st8F`B94B?hjI!>z? zo@*eLc}X`$*_e>LI6{ZD|o z>g}=A?=teuZb=D}1A?lVoTYA+w7Fnf&b>5Ubq5mcm%}P^7HwA_=Qkt-0>K(eC^xVo$}oYWgMEFGuM;QnfHB_srFwsf7mC-a{<$`Lj?tnzG9fQ4CTagGFwA*oQX@y{((Ql^KUdS!dNO%#pd5S#^$B(wR_nd%BKc!3v&D z-V_tz?*0{5sYSdn-hEj?GBi*Uq3SJ6()Z;Ndg8n+^T#*S4c?y*dlT4Kb9!R)ePHK) zvv>92SqhgBHx3~4CiZO%5G=A?d=DuYc!$=*Yqejw(Z#{ilpi*`$#hM5=26f9Rg{O) zbP8{s6d|Hz={3=PZS&*)%U$*Rq{1-fE$ou_q;3s$2_q>NdC``&`n)IM^s;JG<-S1ZT5|JLkqmQ4~$h?pEX2TH0Aq}LGF9iw5+TYf4g_$~uDh_fB+HGqk zKs1=x%aDC$^P##rbR|?#dLc$uqPX3@k@;nxiUR$8$f#(30S_xtHC?_qaRDPDB_V5? zvaB%wKP0tFZyabHJpmIZb(6q!<7o|zw&F!E-HO&caZlzq5jYrT9!bBL;bkS;-fz_Y zH8|g$?#qWqxL|Qbr)kx9#hvn1Mew4jy~>#M;WY#Z>=5`Vx409}xC+f$eiqC?1@>4a z2qyKjYtK_lFSWK_UNZ=vLAV&m^7scf#52kWoFh#&Z+H38MX zYA zm0JrTZM;ftRXB6E6Ov!xRZH=fQXe)+7OQ$8jmi^PxiKOjuLA}qgad!iynr^F;CVoZ zE^uzp#IPg<#W{@34YmQqoF;Ozj=mM;CFym%w1?Fg|t{1M3V%(l|et%{H9yhn$R`IKX zYf126Z0gwWy3-FWT60U;zr>-Ir^SVWcUZ_$h%gw7l_S}8X{EnFsKeaoUHHjYH~w9H zdQB}Q17B$QWY(eDIUjs2m-FkVi+Gxz$f`iYidfJS7*Mp!6lARQ@bOOc5a0LE)ll^> zsqX~$Fqs-V-&#q`#6R1+xdBQS>HGDwI~oN!N|mI-*ySANI`r&nXv4fFTf1wziJsB+ z%Of<9;q7lz&@Yizo1#=p1tZ) zU)IIZ&AxQc(^)Ug!@rZ*I?^#A`5{f`oWV5ar^C0cBBYL*AuXZAwO1tKey-g9dK!yV`6__@BaBY;H#=+T!@kd_sBCd(L3toD|ls|rHt2@HHT0WGgO=m8VHp=yp0Nq4<{EGd zT}fy!F2`?(o`0<9oGfrwp)ol~J9AsfcN97D1UVC`euXaWwpK{k2CB@ux`RPp_EC*) z=an02K)vy`22M~78+@Z-)}i`o>G^N`-*xU94u3;3j zi+Wof_S=ve?)g{=sm9a)|E!K=nXw0U^cnx<_Fo$Tqf$vMH90qvJXo-i^`*_P-P%=(`c?WvNO^NvG-a(_7pW=Gh0m};v`z8>o zz?~=n$UF|H`G+r~Ybz(wdzBqk)u1g_GyluR{G3+iJ*Xx-X2FW?2CZ>;(KA;^bNlLm zjDp1W6*w}CxGAc3+SII&0chMY=m5gVrTsGZ_03yHv+7un6!F-d#4UWPhbc!(0PcSN z@-`3b7ktr3-!NR;9UrQLVs#{h4s*NQe+NheT`FzHIUF z4_!0N=j*Y<+1g{^oZm7ZW-E_G)ip3+e>-WMG&~j~<=D*azGsIQT0gx_Npa`jMN8Hv zNWA4Z(r+n~^F8BVq9GoR!P`xxrA8r&pm7B@2M7J48csQo zuJdJ@00j$l-A*|x)GKl^mb=TD1Iy-d0rPOuXiVSB*+4V!H#;TB->zT!Gv@_P@~YEe zYH)o-re(Lc%{3Ry3CldyYiKJTqL88cmm5HoCR4rf7Ew~yt!Wr2Yco?y@6Ql)ng?$+ zT@AT2$m&DSod{Xrt@D-=-xQ-XeL zRzbaQe>iS5DFx@BP=JwH*O$8hQh;zeim+?tpz)m3s);oV(D|VF-jbjJGm7+63YC(P5 zBCZ1$Z#%TH6th&-q#I&#l%Fa^cZH@7fC2ij;MgwH^u(Z2G`=l|daUI6#26ukbiHrd zCv)x)nX|=n&Cji%#WbG;h}W02f&KOsmCABS z_I+{I^wFQ@19lXtK`T16*u≥%`fv-{078TzjBiiRBhN8i#*g2VsxH4VsSM+x9U1 zRa`-k~57L8TT9T96T$M-YXC^=&YzNEPgX=Set_z8)B!FpDXdP-k9U7Eya| ztP#(p4S2q3zpO--JZ^hNZC}}%`v8!g)B&ZXR7s-{w8!1&RaYS%SKyM((aYQ|zsi~W zD460E?X8}3|JL800Q7+-e+yuLKhtz1mv~is8w%fF7ifmsD+No-=@!nOBGyy&veF(LvT#fad{b#VS~~hnFMmz;xteTcjml#-yu)`l zEYmmB&V#U)L{HHip&1QVc6&JZ6elw;Lb>i)^fXP$)6|Tu+vGZ6xVLz~=O7j|;O4V2 z(`9l66%e+?A!NRTv%2XzpmqBoE4sobsH|~!87R&dS)ZtJSpI<5v|nbn%H>UDFvz#; z4CC3t?zP65)R*oG_X&4g>Cl-fJn`*YIo2liqa6>3Rc}D3;`n!{qACx3Ke!wZ)>kJ1 zYdPz}i)IHfs_E{IN9`FE2UEyMjufcwi1hItc7Re=`Zx@JU@AJk{&(^CyENAg4F@`$ znt3XxkY=Uq4NcvVpE9W~g8r%|{kDH*8=aPc1O$O~qe_{xr)>cfSV;e`!*gwL~%OgUo|IAGO> z?|_|yEh!nPr(OcL!oo8(+}ng2sSbo%?`O{eOW0RDiW_5aj7Y zBc}{%Nof+B7cd?w9V)-jOH33G{8zkBL092+>nfW-h#c?$o-YvPC|T*_{ICf3UO)BU z9aCuhTtVF1?#q*LVL5ymFXnB?o;C`CJ;oZ@MU@PuBayIRfO6;0Zc1&sCweS67|lC~ zZ?1diq(M!+xe=G$HeGJ9?NpeN|`)z3Gztc)cj!w7xjYpVv8_Q@gt; zMz``2s5RM)-ZsD49paz<3Mnq|*u1G0D;?A#F|a4%8M2;-1dD`oZ*7CL)7M_co4=Z} z8q9g$Ia+O|U(m{sR$atNcId--rR z^#3@u;*k&EfDXyn$aol8Xjx)lLS-*z=>o-rFmOC1WZmtY_ugQW08l4>S#0A58<&h| z(%+5#K1X^zgZGZj5+v^1qw)C2i=P)xWsdf$`r72HgfmQ)!pP-Kmsyg6mk&Yk|r&y<@rZzig<&lzM2 zrmNY_t3j2gJumc(C9F9!9TcuaDZ&~{nw|Z!1eX^Q6}FyM_1N10g?Qmd9MUxrYU!j< z-xc z01Jgb+em@({wz4V=;H9qgtOuYq?1!~`P?6Oi7j^E?>z4RmwP4L!sY0Bgcnpj#1Qi& z6ZH&x-8zk5&0DM2Fv$Z|->0g{%*5}SfDXx_yQ$DW9ttfDI0Zo0akqzapqb z9L#h(QhbVd`MA?Q$|_HSF6V<8)kgdYpSTMm&TIzJ+z8+%ns>ME>U#mvg}TTLW&zN|5TBdUFqv!=6qF|gISfO_L^kjG2aT_3JOd#Y z`PUDne)CvMdu0qqmLPEejGNPc73RYGAHhn~Mf%HNfG$ghd~RaB&yDD!W*I$}@%-k} zYNcsu(Brz2CNTL~L4IOxuduOedX% zgVuPooT^ngTPd*f32+Xrud(t&E8n2V%zGptV}I`R4QTjs89?1$@VZ{Gedi4y3@O}V;8g1z{p z^^Kk0fk^awWieDg4#%HayN-+1fIk1_G=Y*dk{>Rad@9eswEvlff9XohUW4Y1NFy(& zkj3yCc;wy$`suZ7QKqeCOSyb+nEA{ZM7!5+sP|S;pP^s!=kMYL?dGiYcY|3*V{v*$ ziWakmsuveLym@MV?p*A(6O6k5O|2`S3y2HF4%CC`lW@xaP?!9e_!Q8mYL*Mu7QrLh zQZ~Yeb4{(^2$pEK0EbmdvSHcTrBvPcvuS8P)Q12}}*$8xP)nLYLDl z;p_(?qmPSDm?y1QpBF}pK|O`;Rd#an zbt1x(r}KhS?l7he{m8C8=S7lEjmC^Zi7_9GU0s{t)gXpA>S>kG2H&q;6TcSUK~TL5 z7a+I?khu0q9ZxY$rJhwC`lgo#Q5`z%oSq@L6W!(`?gaP;98iyHED>3*KUt9pPomZ= z2vbc3Y+Z+|<&6t^3e}5xK+ET?(!c%CTzFuy(2dszq$wY65X?NJO+ED*CVIP1rKw~! z?b9mCZ@;%~C>kbWoe>Xfj*YH0tgv%;K(}4i2{lHlaASAVb%JU+JLcz z=)PEj$e9Q1EJVqOGyV$C-_Cu=af#YWSrO$h$AzEyWOb-bo=ipp4+oC->Wy@gd)Nh% z)b)snAwEqw`|o8CRvn5||KD;Cwy14Z-RYd1s^}F6q77e%-qF0;&3HYp2Zdb>& z(^|jS`ADnxk4}E+lu^oX&V5;R`pTpZh9SDkWMI#*JEPgmfG0Cn2x|u9uD~5=LU}Z5 zehDGkSBGFv3ud)4Z{dgU65?SGxyPa&bn^nah0eKB`$5=u%Wm%HZ_OJ(`Asg zkYADO!DX{U&eOO-U!N(9Px7OYrpwwL5W{bI2F0A3j)uW^FnTdXf5cS!^T zUlmpyp}%Y60YAbMmcTi-2VH@!Et1FP=} zFHP=6ZfYLF*vS}4=jxAg@S6q&S$M>VrbYLu(CJuW&@@-g;=pCrYMTY+G&$9}wfOJ+ z@X6s-r7PR?p(V~3Dft90%O!+TUF}vxH?IMbD@DQ>ag;J(>^z9f>#w~TDR8fCm7)T= zTqP!_LUleUU>xMhiQh@ld1$E1gEpg`@e6Hl2lz?jUylp%Gl@C$`Z!Ib-&QfKnRCsm zqP*sttL}6+BT-9N$f3L%_E~YE2pkV@jJM*lRrTWxiL|&x6)VOLn%s+Z;xZYJd-nv& z#{G>5 zZS|Ki_}mOdL>YW@1H%k)w=iw2)qPq&ym5|G`MR$fmDPjk*qIKs$=E%mvVzJ{QH|?b zuk|FKsHAxD_qY(BkrC4OF3A76{r(diFlq`l+b=ZSWYQ@iZJba}b4#ROa~URcmYL!Q z6_buA{HLEJrgn2WOzs>`KB}u4Ym(|)>_<9#l^HGvek|_fra1-I$yBANlU;Xrdh}d> z&g4xW!|6~Dza&QIT(*W=HlC%|`Ep9h=dtScGg5%Rn_Zx_}HS0j& z>eb$h<%0d*GZf!~xj#^oS6U5BSL4eamLlSISvC3HCEj~jb=&|C{^ma){E;ph?ktrn zyL$N=V>E;*?GL*S)z%)>OP2${|I*!go^@9nBmYpqYt4q$4hbcsoF`Z;*V}Io>5qld zNFRU4z#Y(zc;IdKIoKNao;6337p=`m%Py-B^(?;%yBh}%)!ur5o;*xWy6{fx{XVs$ z&OW%ycdTeq&zQ#%3J-|s7C+fmZ>ztkYb#Xpw)a??Nvg!p&J{U(0_)Kzx_=&SAsL&8 zy7OXEJ1Y!fS9o{5hspH^HgAi$az58@zIb(+yr4b1-NO?!POu=2ohMi-KSzo#{&Z~} zbz*u=1CAG+LKyvt-a=JR1=L4q;GZ*atN&GIUAQIGqqS>qyLyA|1wY7`%WWjtJ{GQI;W>oyBTy{)K{@iB=U?*W^4*Ou#g6kRnqWUTQZJF#q&DcU;MN?D|DG;#37TER9s_^C_xXM6P2bDT zSmZJ)uIMz!`}3%z$pm&TeCFxV_q>yT%6k5-5s^H5jP1j}&+rOr_2p85;R-Y1SF2|; zb5gJ5V+h@%VF|>(gD^e`MtZwlvUv~a5 zvb%cPNGHywPAuX=bdd*Dl!#CdE{}f$oRijRd2&D{F9KZQSL(j~1&2BciYxpp9&?TS z?o#We`je8td$01WS{azqD=jh*-&a*)Xtlk)iYeTs zcds3WSgeMS!gGKp(FR`GMRjOv?gK$H646+2Le1?TFaO(C_pO&CRi>E^kNs9<2Cnnc z*-=b)$*!(lZ(@7in7e1EDL>Gjkx>W;Je#6n)$^>Xe%H+UVcSmkI=^B1W9pq`GL$p< z-*@y%&{%CUnYzbo7DKjuIcZ6jP|}#K4mfb9F{1o+yCkTW@$3Up3<&JE!1YOrn2x1c zdu;X7@~^_odNQR2}jwu5dFY;d}#|2rqNv!w(u9GOax692K-r}2@ks)r-_TDuutcE8o|Vdkm>(5}771o5{}Xfc54!>(7HX zc>P~*P)%Ak`Kl_{%AhPEG{1eg@k6$3iJu)G%&N%qpJ8{Yo< znciJ;y^-EMY?~Hy_ujz2?nS-i#m*|QDpw=$VJGWLUCO)g2UcErfi`1q{~>KpCY>m6 z*}6hS&e@duUR-JKS#|41HQ*z4<9e5K==%0p5287H$yV=}P9a2ZNC;DZveRp!2 zYud?-1v#S>rj;sO7gM)&YkOklN6g{e^+& z5XK3H1EMoC^jmvQy}T^p%r7&Xl@yYsx1DBh>>3Ude@1q!dd%I>pFAu7>VHC*SM_P| z?;+$a5c)GqRYIpOD)`dy@Me&QK28baLQ?W)N{he;#pn;+nrZ3r(az0rv(zg{f82Xc z(Jp(NzB0<|-xFG$`Y?gbuF*_OcO<(t+-th|0c%qABqx3CFfo|?60R!C z&&ohx?b?ER?DLLL8{J8p9KYWs-rQZ^P5i23G%7m7S>PfmxD}0tjPSdMsCU({Y71>j z=320L>DP6yJ(^YYyaIEN z=*L_ABET06Y&bPH?i$T|O?`10yUDHNhUpZM%~;F^?o1JLB_V#4G}lYNC9&QZH)1Na zly&kQ<^S~a6myr+AGQYNfP3{Y47r;q9Y`Y$v=O!oa>VJC9us13^YrXB2`M{J(rK8a zM$vLL$y60vXah65RJOi=*&u!fn9khGaXdtB z?9xh6u4x6ofWh%nhg5C_*M&q3axp%Rfjl-$VM?mMMqDaSLU~n9M$m%K5`*M>e&mFa z)<*+WGZCoxUJ(wFd>dOHk{)_#QziLg#^N$O!)B>8)*ad@;J8RlwhfA3=Y@Mv1pezi z8c*%%k|j+>@jiT$=cFl9Iu+3bbkbICeDPGXLVhg{$}J$x=ggdI3lz2awok}cr9REC zlg)7t;3_lD)>u8O3XSdJwCT-!d=Adl<3&Vd#GfFi>D`@kkJeZv{P+9G3euMwWC|oV zSTLndG-(N9vax2vJr-{x*Kn0>FdCoa>~BbKHg6QK)lG^F>>-?qqJ*gThBIxDc02_+ zaD`nQ%Yv{~t8Ti56-#>!{-r%XcifasKr_sP`0b7e;fLvbQh1)uw71^6m)=&|!gPB; zxY8=zrImLiy!}JH!&04)I4dbOAXaKG53WZM_~suExIt6grq>}9DN1;1WVlw|Re9iV zB!pj%+7by5O?zI?_=l|BQd;!V9LAP@;hGg3PQks`$K>~o?n^vxDG}-WN;kr(p)3-h z(I!%((N<>D`9hKzb) zY+O~;1er)RQt*OMLY)1}}f!g&iG6OF<=(e)T#)RdpOVVq@ zcq02MpswD1XT;$}aO{4wgz~CNp3Tw@<48C=-(k2fuu=fzj_*Z5Wo|)E1s8En|_|EI}Y--SD9PxYep?Y| zGjwmB=Q4b2I>2GCr4Z6#zdMa+U+`FZW0fiyo<2<$##y+8kbYerncO$FuwEx4%O*WB zey=a^9Qj9dJ+0a}9B0G;LCn12zXj{E8oUo+-IZ$l0b6ZRa+#|UQoQ0bS@k7bPIMW4 z#^Y`!0M2dK@KH{BDf*L3f)gobI1Ea>4!g?T^9zocKXO{`H(bYD5NLDBeEqRgB~7^b zl0}&C;=`V{XbmQ!u;a{wz8nsPs@v5@Ek}~$|73LRQ-de1_jByKr#k?M*h|-+A9+zA z<>>S-7t<`NxEb}~wyFhNAB?jE|8-%g1<>2zS~s7wYyTql|PWzyKKdrhDIC#=3V zo?|4imt@mb(KpOozS&>9l( zp4%?fHg_cVG1aytqUFC|RM_Sc?eUi}b62t~YO&fILY*PJV724qB@a^Mdu#u3aApS> zt~b2iZkiikLU&a-52JBTykQ<$E!1Phb`1~o(Gl(=n8Os3Aw&>kx+%d}Ig_rY=ZDd8R%DB{v8$FCzOzkt7>U(Y-TJU7k+8OYf5&}#ra|9FU<0{~ zyidq2GF42hgy22y3kEVN4nz7~^-tr(p1RZ~6Tc$~m%#R!BC|oqNB(6WfyLw3I$Y`s zpWL4tUfRx`<4?%?EI z(@sc6!%1ecIuhKg4}mFcMDZnQK|Q`5?F*|n`4qcK8fHz*XrJ(v&(`}#T;q~Bd&VNP zsvd^!Dm-v>wHp~3CEB3G)5;W$+a8?h&b8UU)}!wX%dOg^pHI4F-6b&1$Rm~_827hh z>+a8F>8hnY*>%%A-rGeNY4wP5!HD+{usCPa4oftwdSO7&S;(1E zvz%}fY5H~-RumY2Qk>uU?3=ei+#NpHyv47C!aM7DtaeAtfJ6V2bksn5%m9Xw^AXz1 z`Ez2V06uZ4Dhz0zj;N$11%z^cBy-^*C%8X47|89oHWdd7u?k+H1O=soG&t(W0bs!F z)aM~E#v2Ioz)x(){`OBSKhyiItk(4QAD+;ylQ*thS<8bvzsmhbk?B=~)5yXAt2Gk3 zIDh$Zn{&MD!Q+Yrg|9Fsh{MdzdBv#lIVZ7S2`m||)(U>efEoK+k1@_LIRw8T_4bjZs-e8B5cR36+O zhi|cn#D5vT?_tQv-yH>^Jf3?5<40TW}OfL zHndF`dsx(B<^9RNj(bI()&XIV&^Z`kjTT#Bu z{&tJ=)zc1n{@X&LJwOliJG$>8xbQN)I#XSieAtirXVuhUmz-8ym;>X^%*=k$LcYEkTXnGk z%i^xG5ns99wcO|wuGduw^X=)H*5JJCWEXo@eKJ@GUARr>A{jy2eoNbkzn0LU!<>pO zNIEVl`kmO&+qy^M*?7Wp>i@n!jfx*Vzfs=25mp9Z*uZbDtz#Exr)R2rrqeEj=!hO~+CJv?vD5hR!pM^YRyVen0L zI(LKK^l>-yvGdWEQj>+>9oa0T9(g*~>Y=LylPn+fSGvNGRVv@asu$)NjGfPyWjgk8 zd_)b%J>k1uRodcL2!Df$n+(Nue6&HpU10rp1Ijw+h04#6ZYDwEzn0t~=Cb_%d>hmf zOSS^#UO-h09;W{n88HB7w-)Hzb{-iaxkN^DqIl7FIm?PScR)O21c2qnLfyCmC{fx? zJ;*V0t(6wUJXNi|~gTT3EB`KViYZ=SriPzAmY9}DMrzfILDpSIYXlR!PQ6OV)q z+1Q8m!7WU&Ma`+L>;YGc=XoguuX*<;nQQWGlw^wtIEUnABYQ%A!<(<+gw7pVJ)y4m zVj}G3*I|a|9-+|^8O%7^?WO+3=seuCGxF99^VeDWWcq(b$V*C~3Keyeu2m*SJ@2Hq zK+i_g8Bp^{>x%zijULFmO{HzUEW$`YwRQ&t0s8uHeTOKl^+*?Qi0d;$1Hf-t1W#73MBXScd0H{@P+e=bKZxZY2~$E zNkiGBF0@#FDRV>Did3sS+4uW|A6eqHRZ^c6fgqquMRV(gFoE5m-J0B%Eq6_5385SU zKHRohL-}{n-EWvukj+0QilbaW2=yWM%H=3jp=;~ojJ>^SAF-2OeB)IAjAl0vuqu}> zPkCS`8;Dx=phljQ9&ZgybWCV9AFHJ6WSr_-&oNJ?ai31o(m#>J^)N0-KeNbI`GLh# z9Ym%h_`q_i&li2Mr56scV4+uE7O9${wAbIw!w{u6zYfP;FpXv4HRv-5 zKB9ygA_fQAeQ}=V?w?WPlH45B1`UX77ypT|fj}ZK*%$@8)QjCq1n4`^GRW`FO(Dlv zqe<}GDvMY>s8)_hU!^iU+u?66=t4{HUfN>N#07ILr}J6Le2yXmAx{DJEFZ|Dfniw! z&^L%aZ#6k}Tx=r(?LGx7_E$QGTBQyJv_0$|mXqxJl5d$dROv+J`S>}0Z`!r{C4MY0 zd#|BaBen)*h2I=txR{2$D%115BNdBm^SUSeyI*lle1hqMo$ZZ`FWi?LL2H4l6)5^p z{Ga!Ajzb~T+SUsJY61M3dFc5hYjhoNzOwZQO~!%oO+Tv`r=~Of`?cBaop1Uq%+e%s z#)>v-FC=&%y!DQGe2mP%1v(?sUgw@$GAIiw7>h5ORyaI!;O5P+&?zpv#+lE|0W3p! zD3|ES=oec{T270L)!zO3zSCEc9fZKihD6Q2;YCYzNZQ1ghs|$y#X5mmli~>zdUAv+ zUjH~nwGUTQO%&r8hy37|Zc{S}J-!WVh&!~xrFXC{TrZ^+6L&By^$G`W3e2xw{pLUN?u;kXuRZyKNKLNz2| z#uQgP--uV55ZeH&_i*)ZNbb#47OpR3@f!*jn^j*CR51J#SO0HQ`>D?#LL??W;Pw1%acbd*f@@spT(#ydWrh6sb zc2;X00@Zu%qY?)@qS zFW9A%C85unLxb2m(!W*v@*lY`lf=Xf^T%_Z!5yOYjMscpj^$J_iWddW)2e5#> zNroGwD^l4p=XOTA7vc}{7o!a-giV3EHG7uEC7`eIpKx-N;^JkyPF`QJ4qKT0A zwccrva+g-CNk(d1@|EPDu_1Uj9rQ257Tu3s0}a#Z8;I5u6t9>+9bnoP4InbYRJI|6 z$%!!}Vl;TIZL~DoqCl`bRp$5{t2QrqJaq{Oocu}4pcMBY-!tXBi%sgEAI+dl*81~d z5mhDZnd}ge>EPDN>7d12db1zJZIAEwLkzxzn)h~{j!#O4FFVciw==koTm?zKYa4tD zrse&){o3=|0l-=;?B=EZO=5^ZBV2!ns80&gC&jxOIFHer(j5Hb4C)#~8aLGT%p6k% zO9vfh;u)AhX(o!l0k@R09)U4GN^$Spvs4+%6f^EX5JEIUS5-y&67%A%>!bI#HS}sE zgr=*nG2TdSz4|uwzY+2G$M`o(Co)+X+O7$OxjJo8m+dONs33KF^GTXhNXd#Diw;Kn$nIrOvj>0fxq>ipV6<$2 z|4pN{{#OA&+@4)}$li01Be^2_#1D~&7Rw)@zYSj38Sm5PDOBP5(`TpO9KznSkcR`Z zc>)jPGLH>iq%$ABVCx){7s@DllUn>S%}k{M_LfkXBnSPBvk+olju(CZCRa?LE+-FK z>@kcbz;;5;xWfp28rK-cDs0JAQy;p0v)wC>!0SYG*qjg2or6%$IOnkXZCI5T1$JeH ziix6`odR_u0PZ^c{aT!dSTgq_Z3YLJ3~&e4O?4|f5YU+A7cXXMSa2W0Wkt4wX$JW( z5`M*MIzC6If)0N?HAHYgwot&2u@!&Em}ratr4CNJQe9Rcl#xFd8fX>PYW||DLuT6q zds?t7&Q*Uq!Aw)fG|VW9+U=u0U6f~emA|*_lI4AZ18ol$|DCvfYl^|wcG3!ed-D$7 z^1h9+=A5I>O<>xGe|msy`vHRDX_Lu0fQN0EJyFp7CQJ2bIm-b6mFC{GWv!APa_fKm zXkSgPl*Gddqu+xNO%q!|WuN;_aZPwP$I!>pw=H=4oRy=h@N73x14+;7E1JzWt)Rg^ zrF{hS0PeQflrxSrd${W4BEtK8r3#8NSU6=_x%yW~Xm=F3+O z6xP{4|+Ac8|lM*^o z@#e;ac@4GNJFdq(rDQtYnd9tIv2?QCgENZ59sXDpZKS@l$ZV^K&f$-)gRe`4E!}3S zfz}wj!{5H{X7}fAYn5!}#ycMb zA07J7*hlC+@T#st4~@~0=4g~%$OYP{z-_!a$C>N6+Dm=5bmn|Q5YzZc0aN63wUXHd zEqjx2e)O9e0Tq^OeQ-PTlK+pf_kfD}Y}$uO?21@GktU*2lqMiZM+8JVOE0?+KtQ^b z1?ffvq$7Rly|eV*qV%P#0!wF+4!e}lrT_0mjrcryzW4LwIB?X+t7R}qC!0)cSJVjj21V+{ZKf}Q|9N}-2-#CvpH+aZ5DNefE_Fc-R$_NZzbXh;ONliw{ zya|}wNiJHSICj?!KGeT|0nF|w&c4kOxD&ic`f4!qFvbL(o^t1;lnghiTl4W(RJJklGYdb1dhs>8zVj`i` z$vZ^}?Ym49Wj@N127T~=HqUr+3gZzT(K!t*!Zn@cvOpIGZ09PPT@N=%Y6-^2%8V)L zx(!V}n3US%ofD+jp=*W}{K-f$NABLMEvp~#~mpvSrj+3o)d$AEG?{gHk z-F>Ex>zx`~c+{u$p`63RM%lbxoKymgk)q7V!TTua%twE-%}EqYqzAo$&y1C@4yNwl z5nUtC2xJ%&wAuELn(B7viCUSQqTHZpvtQW!3dLU`eC;11K|B6u?Wx+@?hZR%0oEU5 z-tsRW`%&DHfgmzS0QiCiL^leP(#&u`{yGO3v)DzGFw6bx*4hs=r<-@k*lS1fwY#vL zZ{V>!HGQ?;+_1&ZLnytVOcV}VyKE?3Y~5tvbZw&d^MK>q&V)a@N}292GcCOGWLPp5 zGeCO>#90mO4Hdq@Nlt5kP_{XH&z$-VY^-^S>%KyDo&RTs;eU$`U@~*4miC%4Q2*3N zn`T?cbNAb&MVl!&d7-355LZd**SVz*ueODY0CW7VdTS*a9!aJ5Vu<`Kjgl=6)g1F= zu46_rr{}~62Mpeq7bI1`De78*^L< zLk!yM)?^^6C7yl0p*b++8h1XDdRLzqxd*M3LI~-o3w=P6hvn4HSmT-~{>9CLm&JH( zIvq@>$qxy<4ou5TD0CcmUgZ71fAh`A5BI4gDB|Bt$Oyd0YX)1(v^JX?Hur8w=&daB zbn9Lo>;)auAbI*D@=O!Y@Y{g)Moa<|6FFpLK<{LyD6+_wbF!T`Ie62k=2i#4mY!|e zbaZhh^{y$gy%4JqNxw$&&LvY)GKHyVQo8@K(amBVZ?k5`55u%Hon3*}rjEd1Y>3@o z^R@a{&*1PzRzdGPr}*GWiVAZn5(q~zhc*H(C*^o~Q}FGyL7!hXb7PzV4P%!EY;of9 z!$)PwGrUw2;EAivSCbu)J|&MPPtk28_sPc(uN+-s-2l-7yqDDKJF(uH7xUZaZvT(( zG?Fxhj~+hb#^?n1m4gxnS1ohhBNA29PHsyv{j1-Q<&nLeN6)PZPtBKl*G)!-kx$3# z3b#hBe7QB3Fn#auJBNyGuAd73C7iIPD0Siiq$r9XTfdM&3p>o)u1OsF)PVuqnWubts%^W zl5H079A~?mORsk)gw4J&vlLfaiaztdY!X2w&0Qw~Xx`)!w+Pd0SZts0@^fhpW#4n3 z4drEhe}Yc2@q5N6dWHtkk%n%Q$bCBnoe%>gn6&Vq16}VjbN`yqQ`xYlACk&@pPxR4 zP_Ny3%nmrmZ@;p^a9ctarI*~1(A&+flNfE<0wmpKpeBCl)l>1x8G$S#>7R$K_zE@n zmkMA5;zB{k_xF3NSRKn(gK{b4H&s}2@8t&%86k^H5T=suuS3pw0Q)x@d@S1Xr`2RF}8f} zZ099LDl-|f^+r7_%xKGRk^w#Yk-pLMiGT1A*}e0ePf_Q>(*IOyRK*69{ui8m0NK1~ z+7#{{qoo5%kXbWHB|O<1V&RN@N!^A}7RFzHa`b)U6QqEiB6VQP_A%9Ppj#Sh$vflu z;{@pnJKx{F*He_>^FZ-Q*MCJGFc+&lRSmYS4j>0RaEkGA+!%9d?*4cdXD<40#R2*d z=EGE_uJkM$o(Fo#vclH0$xh#g?~x7sPDB=ct2kBW1{2-C-4qs*tW&^| zEpC8t@u+(UKpLSLihJr2Nyoged-tcA#0P`#tQ9-N3HLJJX;tSgy5VJMcu}Ig5e-#O zBNGQ~&IK+6>%WHfEV+d0Ul6er@-0cZ$*Yjfd&E9>ku-PYX{h2-;xcM3QO0PpKjWzy zxu%R-Ft!(7Wie%09kHBM`cxrnC$CpDIWTQ6tY>R_$6}GX$V8X*`(>KDupU0jCYAeYajm@2x22zL0??qgEW`s_LP?;D z*ROFqh}vE^X3-MmkhRfthOt>RtXg+wN%d+>ImU9RocOL$C%XDy84F=OjNFH-W6$y^ zchs{~J@SFNpbn_r`Oe7Uki5%4%si_n1;Sd>A*IcJ7BV+dc2bxFzRcJ51qw-+Vs zZ`zxy#~WEHiJXN=PE>wEr_4P01cwWnq|AQokR%HW)ojVd`|5RxD$HHr3WS65S^<$>ub?`G239q*r3@VDM4b|y!nd6 z^te;}C8IGOh;8O)_#0t|9mig1BSl6So*%M*3R5^Mm@Th@W$X9~{({UoAq5u-pYLve zo`%>`tP7qW6*eV-*Cy5__l>&T>{XBD{~;D4dfoV6X&i8_^N?cn;bT+PMS^-O1t_a4< z>>_+&$5CU+NEc$@k7AaO=OAD}g>f`5d>wi!Kqh!|uq)?}z(?m{=y0!xwu=s#qi_+h$|uZ(;5lOg3k)}I#825S!E zzVo;52QpX+QrVIuKR1}UUiP#~{u6W_Oa*L-17&EgE1R;U8S(?hw-Ow$6;b3PXHndi zv57Z~oYxTk3PHMLWe0i{`ql50(H)qlku9p9j7xtA&xnFG2!LnD-zt^6CvvYG6A>mu z^V!klH}ND9c1XfKVFj?@%ismAZ%By6U5OdF?FUlKQ&}@Z)!kZ2JS3{|NlM6Vd&I4^ zYscc~RU;x!4!qCj7DJtGbc(-J*%fKBYYOo(8y=s~w%m6Rlf+yv5ZmGb{hp(0TV^vB zGs%U+gB3$j;-&rKO`jVBK zZ+U1_>$#mE*Y~$`)kzLpoV}NuAT2iYwD|()^1^&$e}LPtd6u^~L8vj;N0}liVnBD! zCUFpz*ZtC}zcPL?xWVYSkGT0+Nb}tQi?E`?NkrvSib)%8Rs^3XqNKygy2VT%UEO}e zcvipM`U()X_@VgZb)Ss9VlY}rv?nlFAN!v5`|RYtoxWMlPM3{zyWXc8%a26+{;^5h zi_M+9jjQxQ&f=Jl&b9JpzUKAP>yW6zkMEQf;^&=!&CPM0HJ&(PbpRE0lzEf+yL5!6 z-z9k+oa5LX<`O>@#C(m5_71Sl#SCG*0~CvF6Hyh=|(8Om}d%)W^#w%aYRj&d~S(g_$00@aq` z!VY0On*IuIBOSJmS9;%0^eUsh2F;qLV*SkC=}vVOV!EYS(RVuwO+Mi6ewuweG`A@# zj`1NCOTky#1W>hHYlvN|`-sW+GJsJ!y?vY(__f{N@3#VE+L z{&SqE2%{t>gAl1;I@zk!2kZjen)Au{ca-qPfbQ*P{GmtK&`Ceu4j8KBk+J_V4qB1 zr}Ks$-&ws4z#doVKMxf{&V5aOFfnz7M{P~>)9mj~lfRtgl!Pt?Q)wdqy^=qbm{HF( zvlvi`W6Ff7&x#xWT-Bl#|FCxNeTb>yq6AoA8#qK#?6tA&Sm~d%ibz`&11-n?yFnDs zMU%;hRu=|%r}3t(Xyjls+r0~>cTNiVae5%OTg|FZ$Qx<2OrP+Cbh#tYV0g4^+A8a z#*I!s(hh#ns6yy7>ltlk^Dd9GzE z+twVGiPw9aOZ^|4{fU7kvWQ#J6k1qf6L7ALuH!Ez@ErL$Kk{|QqnoB^_>VPCrvUx* zu!`;Cs7an2UK}L1VLgIb#|RhD&nZcDr|B+*mt)9X1{CVUFIc=a8gR41>A3|xCoWK^ z6FUwzhHi=)bn&QFPEzLh7v7i*6Yg-6n|{KUkdX+#jyUPcuo}lm18QfZl05qr6Yo~r zu7NcF_P2^DNb^yWppj9Rm1ZR;3sXu9h(T;D;IcFOpeesAqya$^kVt>84Q!_byodDL z#i$#IO@9+a{D>#d<&rur8Lo;=v~c^j*$SL_Z%J;HO6qxqOyF5a@jAJ8!GQHytrPbg zu5c?nQIu64^ykoq?)$uHCOb2#bJyUS=QOP`Y@#C+OgKJ|7kH+JAJ3*ubD5z8fBgG@ zwhjJXOW^Az05ZNQ`2h2Jn?oi$ph2+bE`u}^fW-f~eTcMs1ezsH?G$mHXy2I8QNNG<_lb4*)W8Rllfhp1#S<&6l?BwoXQ&JgsO7qK8Tf- zz{TRdnU>!kqk@22z9Ad>JnDg>+Ra}Q%*SeBeuHwPVG0`YT5OY=Y)|QY>2-91Gw7Mm~TdQnSgKW?8AC{Rz;mKqhm8ZbB@cwg?t03B5drm;W zUsMtg7`|L^sy8>0+d7ZL({w@NT}OA63xNClkq(S#;z|=NR>*vd`s>$w=KG*Q{a!0( zonaALft$&Gum;t@f@Ac)MGDo8I&s0If;kecS~#1fm=1ftkFdx1DADzkfg`)}8d*d=_^azSC0BkhePE$<9RN$XJ<%$W@YoVEr=Xa@3=J~Wr4AjhYD-2Lp;ryQ05IiiX zyb0rCImKc~q!FFW)?@6R)Ql~mw6I}<<^>k>*pMy6Vp^Fl(*Y2<^ z^R~KF49##VSO=sd%aYA!n9d&kz-7V8A)kpZYwEiZIhP=WANC}d1{`AY58_aoh>8FI zSl&ljQ$3uF?H^J1SgF>ud77IGY90uY{rlv{V7_AV3F zblV3!bnp{@(-V=!{m-j(-lERt9tnh@;MTN0iN6^%)CswnYh%caN2{lO|7EheJyZ^1;|D)&#jQ@xkdaQO5 zu_X@vOtN&<^R|ghR7F0aRtcV*5F>-vEWLu2ty{LLOsh00IdRV>o?e+vZDS(*0N+q@ zuw@b+KAi5{sj4DY>De-7Uq>k7y(m^uGo63qp25_K!|I*##GL9p`b6U&btKID|GC;T z&)wQlT6r3~mqCyI3TsvPKfdfOqGlp~?}isM5==?90tx|&>(!SNaw?_-o~jGHZ@hSW z;cl<_wYND)CKkeavx!Iz?XRxIMKySJ#UYUhF1d^HGT5<#Irv6rnpKW~T8C;%^lXl^ zI-AN?8h4fBpH;;s@jG2b4!xazV=6#4zga>R5=fWM$q`n$^W?+T6Y)_iPJQEXCh?Hv z9nU3rMT@Stc4U1@UDcEvyO$*+{+5NN_rl#Z`UG9g6WfA*sSD{%R+&~ABFg~>l&yPy zQ>9^T#Z*1;#*XY^DPkY`dRki_FDUoqljeoNPWO&z%8N1&P*O-7->0v=I9@oE)3-H}Q80PMWNZ9txxVVF96ELDuWSii z20kLEg$JpEwuI#ym03Zo7R^=H9LjT=ob5B|O;`tNVYS7!F;1?#Y8(3ar`*;(rW@@J zF};`iSRg6UGtOT1`0F)&wx+aC>;5q8(@Qsg1tcX=N>&Q_4f_ZDg3&tO#@+Y_DT}pS zeR|{Zi>6&j9xECh>HLP&PP1P2{zqA~DE^0N^)amo%`=+D#Ni)S{X-9^T}?jZ;3Cqj z9A|o$^K^3EHkZzG3-52cB^i1+mAd9Q3G7!!a%&Vh7(3CFIp?P>VAd%GBavKeuq|V} z+A9kx^$ulS-PH(6zC+dJt;aimTzmtt2a&J*|8$BHVe`zoP!5SGCg(|0#`{fzh6atq z$@#+SAu%*E&2;;fb2;$$6+93F5)@Z1j9{wjIs^}RiUIY94@5z}x!V3i4pu6(oW)UQ zwA?hwX{Q!$`eDf)V?XzrA0fRxcJsO9R$+>%;jHBbs!R^YXcp8Mh@&hNk8KaQ6%%5hd>Q&gxaSL+d0%pT+MhMAF$OCg;JPkD z+Z}nsSkr^+_;x71IBM8E>RU}M3bvl-P$R|gTfXM~6@V)olvBj)Z z5If(Fgs_g)z!t?_`4mJaFW|Vep@pS3+fdIuGu_V6oH(W!^y6L^uK?z=>ah3BM@INN zsbRr-o3{gZr!sqqxk}8YL)K(g=zYwZ`!?NT(;lOU0}Wp)cCb`#-)@w1Bkk0Uud+UW zD#_D_^@uc;lylUQNGi1Pr_O##sCNlI>qN6jEtEU6XsIruT@Hnne^JfX?W42(JuS*- zdL7e0RGz)rpnpp^T=T^8;cQ?ao!PCc%DJ%&_gyi5KAfZF%o(QF8Gicnu)xydgk_Gj zfa=X2HtEi~7u>s~o|cp2^<%=bM~bCZy3jOYGmcEWR+1j8-jU|8p(;VyQVPcekIN)nK*^er$co9ECULA zwKxi@%92m3RSsnxsf8`_x+u~JT90rA05#=Z0h`sgL(c03PUY2NLfcrajZ*qzOmJmS zZBD?VATvxqwjF9EEVKV=QE9ocZE+sX8?56^>RLha@pCAnYRl(4ru%jB+>Y(l*tU0K zGqPLA9s6mmtZwy;v4X$h8(-5wKeui9knL0fzstm1)Bxk|X%3~E(pz-R6W3^}d!sIP z>X7uO_P=JX<04>v3C1II<=w3p-3?U2XJo_DXdKcGu#bzrcS3$=DQup^6hM7B|2*~H z&xcL7Cx)TpCAqQSU>TZmxuC2Ppkd#F(l&T4tYX5Sx>WpuDSHl@h2)O$M*HRu?n|HdN zE6$%)j$7;86 zJkEF-skpYEgKrwqFrTV&;X~LgA$d0Z?AUZQLBH>IrX`_f_74JR6pJZ4Z$aE^J#AMj zos>V54F$c=JJxu?`}13;Ljt}v@&155q!5Nu^8mOWgAhbFhN)ra5>A`6kEb>DdW3_9 zyDNu6vda$3U5M+c&J0kbY0!UL8sL!=*NVx=iO*#3`>3i;W-JVD%Y88L1<}Je-CH{x zfrgOsNp^(K#&&aHrS(z1`| z?BiX#Dz2zqOk0a3_+1I^7^bGz^i`JQX1L93witoxp&I7W_A4C0LKQ|?RVFJ`ZcE_? zU+|jVr}DG3Bhe5mODk3Hb~_p^lBG4IV|}49&4__?^g3m$RM1vFG*b;89`t@dT)qL? zSExK%Z1d-i&CZtE>>{NyJt#x}gTp40It`+jQnJ=7KZ*lh>96li{6FA51qaN8OD)QV zy9QAtXBh1wXT@OMLaM$QWl==o4h61YJ2QXIukwmS-Z${~zCCt;+<^GDrLJfKEdhv5mqyN2O_&EW=OtKZ=38p+L{JJ_dD+TJh2?tOU~(^TDm3LAfbnD~ zoTkJ7t%PT=qM3Rq-X9ghK;eButi?(zuPTscMN%uibv9*k?h3bkgY+`W+SSR38@y-w z|93%tPpB?=gn798W zSVdxXrW)yV0q{)9Ta6F<92cGqQZaE$D#WSSI+SKXI@d74d_8dnG9g~+{o`Ki{DM{1 zX%28lrM__uCz|VvgUF{D{dxl<&vh4FJ7Z07G3{72cB|oBl33oQpn^TVmwOdX^l@wb z5glp`J!W4hwbRc3T#rcBfDv2I7kY;BTdDCTZ%CyEfdXQeTi&3VIHZ^3vxQw{Blm9c z?m1tBJ80Xs;f;~It(P;wzX;ocRN|^28_;KURaVuykbzh06`gL2yr4KNf09O~ow}TY zOIuo~?#VhkCx^jXxGN}{OSg1?0YEBzyFr@D#Y<=HlX+SVz$;7cB>#5l>97ze=Hfo*&W zkQBDAZi;ndp>M;qe_ZT9_b}8W_HD&0Gzt6=C4uz`lM?b#AP&=i?d;UfUpo1>sJmU; z_yAz^99!7nhuNOk@NOx=aGO@JkH~aK+{PlS>DY?FL2HP|)ZWl$9!|R*yXkf*ds+qL z??j4rAqPh)faCy(zHd>d(rv1qzpmXK4yIp*ZO9HbbBsPk!CZ%2PllV>v<;h+lBuWQ z5tA=FGgzHR7^>t|#(1oo?}s?CrHVM-e7KeVM*pndWO;(2SXPUA=79vmOzc~|vPZjT zW31jY)9!a{P_)zEd|dY{#yEAS<8_4JM`FF`0^v3K;gx6m&<@YR^_1DWtuB#%K1!r5 zpAZ3bg^F(xryK58;I+IV(K8K|ZkwGC)ODl~5lge@p$M9aQoo z!4qk*E>SA-vHWQf3xi2Z=FzhOtWu|iEqK9mCJES8<5RVO*qF*(gA9k;)6d+Y zNRG%D%5Fz#yac6VFo< z14L9dwE2E<{5EtbDm~|xN4VRPuf=UN99A6A)fs5jXPuN_98=CleoAk~v^k=}-u=y} zu}YUH+9UXDRh-bxdp*_?YzzFi+tl?3EB&BR@QyO%K(cu-Z+E)NB=hq_2BXRvk|+Hf z!}wV7ka{1tnJ49iyY1ZnY$p?iOl06!jH`w2fd>rGBw2YLCW~Xj3qe}i?1vZY`NPg? zW|WnB#NqD_70wTd7{w*35F{gHVbF_KT7#5htK79sqC3Gg;`X%sx)zr0XGViYy~F%u zi?}~aIjg(Am(n2dykSr`hgScj^=_dOT(EF;Lr^98Ni+jb|W2(BkoYmk_Ho)8cz#$6vhnS{0Wd@^Wif$$2 z=8W|iI6gcK^D!@umeUw;(fi=vZvA+M5C6xe4mMHV$thccIKL#HUq`HXfec?Nw_ z<1ZA~W(Kcsd`)gmcc&V)`c#9kHEZtU@EAHZ?n^&K`HzK%o#fu#yJS}i+x;BaS49gi z<(6F->U)r7rVZ1&7`kbwBQl85I&2rpYY-`yxVR0kry!$9JH3yJ?-V=CD^6Q-h7IEq z2KpJydLCq@q**F?*Q>=1<60I>C=N86_(jhVh*8aJgGG2cZix><6>Ug2!#;Kk8r259Iz{5ldxv=z2YN;iuL^hq# zx+ZOJ8<7Ad#HlJuE|3*~O>$&f~6Irm#u^Q=D zb7$IL6E`*+^$Cm^XROk~E3`W6N^X_)qUW&KzpdQ3m^huwF^ALf5jzR?RyiUC`9ie} z!#D-I)c)+^v@T~>yxq&$cLs!QmCm0Q+36(mJxx&OsQh*Wtcj>q{CsQ%MksLc;TL)X z8_v`+M=|>tWXNaV)5fayce$%_cd+ZOMa8-?&6EwRcR(z>NitQki`1U<=?#^2VoR~e zl}L~02l=qSvv&rouHpVT75mtvcxz6N+g_}glGC`#b}t<0p?`YMtIv~*IOJ6Int6L% zX0kI94108`-%MP;y<7k->g>6jo#$p0ly&)SQy>?EZQa6+D#6@ZMG^+ot&h!sL@tBu zdzW5xGMZstGj;d!V6qc@?eYy$D_-Rv@;Xr6D%+8~v_ze>X=SAd)(rmnbZI{8ex1-h z7pr66pjLfJwZ(uZVb`i31?(k&-@>qLMw>*kbR($>{C0ISr@SG`Zv4Y1Gy}pgu%9f(=hl z8~;d|OQ2Rkb?tENaHtg##H>G1 zhe*~*M5ejyq&n2l;P+yVT|Y6!ktb6Ylqhp91kdj%^sT$`g8guqO;W<(cB6z1#x=$j zVbc`Bxi9FS2=>cd{v=mElnL|op)1DyQ_Wjfl2Zbg=37+Jjk#u_GRHQI$Li)vbzI(t z?$VIj75@eNmD>GCTs0t*>36 z+34$16X`*TyP;@7hJgWt)+7|X;(WYC^12>>lUYnA(lq96>-B10(;;Va( zI(WozjMv~7uu2v6h-4C4=TatyzY!tySf^+>m!3j?=qc+))3`Q>uoPWx5~Y*(S! zH~~#7lsHESGmo!e~G5WFnOb6lH697&X+p6M;@vlpbQUJ3`{`^voYGbL zcL|fW;haSGv8q#TsV<#+7u~?9@CyI3XX26ITBez>G+o>DU>+VMxD9UU;qCG3H>RJf zRaVuExRIZyJ8qT9au895`vE5DsFug!GVa!AeYG=dVB6x;nF&4%HE$~xVHOP!CVdA% zKq?r3IGViCpIfctghV*GG|lOXJepOEtkfTRCO@+WfKks~%b&Yv!(T>x_)1-SU*9#) z0l2ZMz%noPGIpAk7`JOo9MTteu35#EVe0RLSC59Oi__dpTAVjdOd)hH{>x_MYk!T_ zs&eo?BtJl9;0La7XFL9Be3Lm+`SP@iEZ@cTxVYgx7Ul#o#y|o?*&6#)bHc24BP+E$ zFDeB)+COfrv0XwllwVi&s%*Dotq$?3FnzE^nI8Pe~uQ`yB7*zY*%`YS|wmtHbuZk!*B zr|O?&CovMBX%~;IUbVY&?Pq|DMY8t0m87J(jn|c(rpFDb)u-pc8)qr0sK%Dn|Kvxpge!@}zqJ}APAy>h*B1iw;1{L^IOi9Mhz{4K#6?y3mrpEW!?9+- z;T$@n$mDrh7^}*Nx1SlezeI~WzUY5eTA!x}Hs{LhcF=FqN2=4GajnQlNnXUIyt;v_ zM3wZ60=4>>cC7{m%^pPhwXWm~H)f<#Tebgwt4SDR4ReP#Iu23<4|bb=*s;vwE(y4$ zPv~6!XKC)slDqlisYN;|VBWj8) ziotNda-kCmW`VwRn%?ZJuQ1x)LXwwu5vt_HL*YOrDE5J&;rn16|ho1~wXI zI(d2reJN8k*{PD7?>o+qKnqh=yXW|kHz0h^sEXxz!hpxc>3-Ts#tmiu!s{Hk2oZuz53q{T*dacMlhrac(Rx_ripG4oO6hw6w3`0(tH}W}Y{lxA^__|?D&Mxa>E~-0 zlMWuHMS$k3M7-BVoE7#BqPoQA2~I|+ntfk3vicKxVru%yR@!8%nGhN%(T8JG9@xbd z>@85IBD+LCO;`TW64Bp8t*g7m9<}1JdO1~>3mH*z$t(ju2)7BySmW;30R4P==)|gT z{jG{NHTW8B+;!Bwu;e8k*|6z%QB2iCxD0qnZ`xF^BA-tPz9y;nSXV%}QF0iwvz5kr zm4l~M=^aLEY@t3foB0EGb9YAU6F#E`RJPDXm12+#SHZWb(Y=>r0nNMW(O zjZk6l8RBtP-+QFe8jd}NQ;3K>n;&vk`TG!sdmS;KuS4!IpaLVRJkOLp%G_VPqvD`_ zf&s-Tj$B(Zzd~y0iu=^{KD(L*yfc6EXQO0lw``bAxWlAxC2kRfyn38EJK!X9IPft& zwn}@-mknted71Dpv!(Oj_mxnOWHei9%u~4dIo{8N(j5?(;q-l&8v~|&H82i`4Zol= zT0_xQDIE=>%8w-HH=^?aJc_fBpMp#~7X$^z#u2{!iPSMw zJE)7^^ZGzwfh9Qt9a)BoYYFcuBDi4t&lfr2w^c{g9nI9>OPj=DVD7S$YiCSa{Ad*x z+^;KQD;=9#HRv?6x)6q+cix@o^^=dPSjB*E^bHf=khtR?-q2}SKEE~xB;pQC8z<5| z+V=&)G|j(^?BkaWQ?&WnS`;eH2`B+sB;FNyYBI{mVoPUZae8^ZBS^^Dm9DaEN9KZ# zYA8?sO7UxpGg~>@5p`)#(`Peo;BPe#TCqDz)|M7y&_of7qiJ5~X?`oB*Rjuz5?64h zo{s0I!3(sDk-w`5e-A5xV8Ej`j(o|?OmQ?G*Kb$aXOPUoRpL`rT*^kD;pO|^wgJW) ztYHcKV!p)~?ofiZjCxK@M5DGbCP5#Z^)GC zwIJlAAyj6aMqtU~2m86LJi~H7pvPcIJWI>#T(3KKt9vtNPDuz{EMY%h4+H`Adxv!5 zl$}K)eUITw=UDz!x`;9%2^JJ`Dbwbg9_x7mwB3W0A- z0>sb8llG@*UNz#i_!j`P;dXN(GsEMm_gVfJTS>w4L+k&BqZjbe@w{mp!8*;1zL?znCX2C1m6#oJEruFepK9e|*MEF`JlX00{z<+8OCuQu7jkCZp`(c@U zeO9CSdTK)1n`+Kc*wgZ*#)FiVvhoWid7o`P{1@l>quu}aFlr8neOVv-NI&+6vG2sn zh0R~u+SPGceIw#%)Gcu$0m1usmocM`peyR_Z!ziqv;|hEuLs`o6z(gcd3K15zl%DY zi)D2bmQ)$h8j};@CKBOkwgL|fpB%C(GwSSZhOLkvzCykAGhr?A6aTybopU{Wv=8XPQiQA|Xn%+&0|q5&(KC zep^e5kJy~VF0zOC*Z{Hz_o7O-^_KduNPm$i%n#(Kx15g{ad&XEKV}!uXPB8a{IpuE zNf=)Y)0vC>c=dE)`;~e*im}6ZV+LZ4m4!yXAfc_s$KeNA@raj&Tei4Vf2iRqmIXYD z|K&xbHloJe$YrM#X8-hNZF`NzB$_2Age^1yS(Fue+9`x5v}I##v!n4hfYs*QSHirz z4ueHtnsgwG>~dc2tLjaPn}?gu@>?+p{ThoHEp&41|G?rw6(H1f9vdg|2qFwzzz}>Y zIN=nvTz&(Hi4xq9t`}p;fdB?lJf1DQgKO#o-Q2$s+5FJV{WEMu(WX)S9wX%D2**co z$)v(6p|iE?<3am4rD?G6yrD5L z?_B!Q&mKD^69&||7_}UG`Q!yr`NzzpsI z(i1(t`Cm$=V{QWYeKS(S$w~J$yn=}RI6c^DcZ=1pzj=2mQ+Nb;elQ%WVCsh3AqLsh z5QkWQdQ_=wi$l2QG@UJ{;a(FXLl9MgDX#IiI~lT-518*5_D9eGXh3i`Z2#?UsCgW` zoa`E;F*Yv~4#{14?ItRmjQh)y0UYl@Ab}_Hfj{7NS$!K_og?RJ_?apj`HVrD4-V<0 z)bRNIW1Bm=hhYx(p&0Fq_ww~~U!k4I54v5&t zS?zWL?~$Q$h(^x#sZxruoLO-DxZ4+8;CniYzoXvhJEB`I4Mg|N8l#0pHfirq@h`9M z$l^^cp3w~PDZxwoSm(?s2xoErm$Q)OSpLOq+G2RNc@m%7d#IILPeAS(&6x`hWK zANw^+Bu3EfFLZ`K?aK$alhQrrnE_r%&VKli{u5$${OIgFe_b(rn41cbah6mW9oKXt5sMRHZD?7|!!F}y*m~$Jt;pfl z-aN4c0aa%u^nF%ZjuRgo?Sf+!o&E!M@hvXRY@2?LKf(?z;~u7a>8jYhhh%NQ&flQ; zJBCM>jDK?ux(Nxgf~M8g{9T$HrD=H4ZG|nPDB4^e-6Yg`!G$!<85?qp^Zvy@8gKO_ z#rv%OwRr{i+A4X3sz-dgS9$*shd6g4z8!boD!&V%{)Y_Bd$)wFNH#8ckA-{1c*hLE zO&;E%mFd#n`O%E(4mf|hutbr=AR=A5n#K0_>3L2^RzptX`O4wF5mNGm41FjbwDFK6 zMCj!7j>GO?4nWTHFDoTTsUF#ve0Bfqop+`R<6#!zw5H55sR7afT#U?^TF)?ct@w4E zBuO@Hu4ZywNg}VhRC`t~kV-w{hA*sgy@rp{;Bw{A;5ROq^z z4Gv2PoPx+3?su&^^bQMX1LGV7p`yCGF8_Z5!BS%|m(1Q@aTHV$BxOdUYdOHgq4e0p zBoNs5CXE@1c^Mc)InLmEs3dltHy3d3>rh=-W9?anyX5OE6v;ZnjGD{aol5;QjapD) zu-Ir|&@l=D)n94%OfIIH$?Qv34O0q|PA9?cfFcDA;4k=&1vYvFlSJvVta6YAG)+ET ziI8T+&^wotFH%}512vPn2&{Sp{d=GN@&>U>6K4@mM?%(9P}&jlP8ygTXbZ(*K*1JZ zo7n#Zk-#tJ5-ms)HR2OukPGN9H{ zZj)pxVE0(HI&R|&z;J{@{?{4`2~b*XR1^f#4)|X0J&{nHA;8SQqA{kR;Pqb_?=jF; z*;NgSkK=WlfS|SXGfs{b98bOwk9hFMylv>|c)i=P6A!MSoYghi4U$r1{iBQ{4E5)W zdKRfydN|v=9`c^nWxv^HJi=)!Ymb1=?=LhSRQIfL7Zt7~h|DSEk#a!N`wfS%d2oraRRnP@2z{N0Ic^)>PA?Oj%qOL}9||!N7A=)f&YF-ivuLWGk-ew# ziBOiaj!|<~Hzx=4+@AIkEEX&9Y0+4z_91*h=~AxN6on?t_CFG?W9O?6wjV^8+IrWw z4p$9h@ihrfC5Og@XpXC7Y&<&`bAmCJhZwhIk62rOs;;S(Waw7IH8Bh6TXS=nDb+Me z$VS`~h>OXQt$)-Rm(k9qb}#~$jpzN6BXR(SM9wJWg(6wSgu4ehU9==+V7R}LW*E2s z1Ox=uB{X06Yn7ABICfY0 zu-AschA_8f`Hg8F1V&@(n$W3E2<&G3)|jUf)MAYMf4Qc0IM${r2~1PKXq6jfG>C6| z#Gv;Bz&y=zkr(C*Tjk|$`wQ+jw)`$Um8W?0$Nrjv5vVYq7n}@}lmw(E=rD>fF-RJ; zT&*||T5Nj9>j+Ip!qA2Rfxd`9k z!4UJo!J*qukNf_X?%sm#zT@OY+5;Nr0|$VVc1&P-8{kTqAA8J;Kv9G!UzX%XtM3NV z=9unc334<>+UslUVyyGZ{$b^82`N>9v>RO6@U5`gM9^AUuNL5pkc4s{K6!{5F2B-GxTzPa=IzZqFcH_~OK6?day9CD(Vs00DEIXgPYNeuWj@D4G^Tfnc^xYVmaR+hH) zD55U!`qCgWFoA#i5MEz3;5rq&Ghbzxy&0z-YpX(hF+rd~Ai$;#8;6_A3I+VH#BS0F z(+)2q?pN|A#4ZF!JEqA)s2voT=Bw()AMA}*+lWdbOP87d*H#}gV`+yLd&zPar<+8+C`eJk1w857WQ)K_n)ZHO-s0c?{YP5neJ-{cZ z_hh&=b8!t7LK&KP4Kv0r{oN62h5l+)YC5iJx`z6Y%{_WpR)UqOw~TgAb6N{Wr`gs_ z`D-Ni7#asE>r~69g;j+%>!j8>u1R8y*|)XfQit`BNRqEHlO6}Vw40;r(?KO^v)ggb zOLJSL!Ykm=I>&f zIyPCcs$Di|#osCvgfM+n3kB)|kz(A$#-cVR59m{c^B>}G?PF`c$`K^^@UK(nL~?NY zKv9DOFgtSU;io$AF#kV#RB+>Y;A}e@0 zm4$(XYSyUu(x0s~~FJc2<6i=d)c5$-%_u z!F5EsGrE$TwLONa;T1!L)|tltkFYn7hkEV*$LqAODispiFd||s*;OL@mPD4r82fVU zjkQu_FNCp;Y(p}#?{rEdW0*3SY@_T9ri?MR;df1SI^B2o=leU4`>*pjkNaV+_jSFN z=j-`=zVg$T@b>=n!q%)&3Zrpt)*|2QvFxID;msaYSd*NXHMyes?MkxruK`6I0bcr% zMGdY&h|kVWJ`79c`kBvDpd^?CLIJg2!{X3+#y~sSARzWe@)P4=Fb(T3nRiMxF@$

C=36A?f~Gfz&=~yWzYkA&cV8g)C@fZ-w*NNDpYIX<887UCS$#xrYjX5NL<)Z zXG=#5#MJ5|?9|?<7U(?0NDQbJP}=r9)*9v=Lmmb7TtIYG#x%*s;po}C1`eZ~26t1h z82g&Q`8GVAd_*hlNad_Vd1garcUxlf(nv_5PDf}Dxft0b*P+BzP&jM9>Z4xQS(j&0 z#APH#Ii}SbB9Nt5uFWkwuBsz$5ImY6_q=^S`#3)bGiG&UobP5ec|qKLgXk4Jh8|w) zu0oWA1T!#P!TpxRKq5Q?c396e3NZF$ng>$KCHX6YktO~5Ll0}>|P(&NxsISK^ zP@i*iG=H|ZKut}{{lBoYuNazJkrl*5wYpLMhEBi|RFJo3ZM3xoBGiS|5I0DLvys6z zuuIZj8Getg^(_a`7iZ)wP3t{4ruV8nWveX266XDdCF5L4s&)MJ8 zzEgI$;FloRHkV2ls_v%P`cpl?#|IE7M-kA@+>|D9w3`icyQQh4dpq^E-_#nAo6Y0oVq4U5qGB1U*8dH~F z?O~9{6~G07!s|MlDsGC^X|ggKdhN@aB<)bfliRD6i4pg_zRq4IPsc0n)O1bKE^4iC zOl&hAHT0XD`0UhKgkoSHM}*WV2c*~GGub{oP#5)1gEWJE!b@hMi*?a!2q&(fgTd^p#P?dBV~^hMog z4c4D2aG#jP$BWe0TE%JI_eNWRzCO88ovjF6-9U+{H^;X;Jo|K3mbA7#! zjljVB_lp$jJ(JXV3Z}E~K<#v_l5?x#n@`Kp2KwAP^YB3uqmIy=6i$vNkJ|q7@d{bP zH2(3)X^BnA>p9w%)x~@>T*^M$JezohP7(I2qZRhbIpo_?mtb8L`xC1QyNZZKFD#2q zJ)im`7Wgd{6a`3^@@Ds+wXf6OpOS#J!_ayKaM8MH5PDz$<#Fcb8g;eCe;nye7TzT` zb0{>kNSz68%NHjrR+t!AV`EN zPe;WHtD(BUSQeC9X0}VbSJTq#`6pF<*`Osr7L=WQb_5vD{10(d*4U#6&!Ms+4qaYl zl_5Vc7!)VqBI+14vE<)tq+`6=31Iil09TUeW zYlG+ruAC9aZE&&CY^(eOvB(Ag%Eh_MPBXdp(4*XlkMM<6Z?|pTv(qRedFg{R8;xeg zuS6z1I=xPW*?^w)=Dpw?UO=+iD6xc9uk9C3@U>B8PIT}Yqj>JDo&)<7)_Y9|OC1S~ zq)*SNoAs$nvw8b&_wHvVHb-ya2fG`8hB9yVJO>~>bh<$cto+ljelvLkcU8ClWV54> zDXLD8@|QiaA<{COHP4AiBi$Lj&?=k@Fpn}fq(4=q8s#xwn(tiVmlsr?GFk@rmM<&;2&ll)=%3#Zja5q(=P&n_g{2K zQe@rd?@@9gP2%Cfd!ofaUCa8xntfC)}_O3RHdT+0a`B={BmF9x^}fe0@fBh zhY6MBh>Hc<(GN5wbiH(&<~JTG>ugkf41v`7!^>J|10`3`#jIE^j`!>4 z2rAIfktL3-gS|EP0a}I5(&U7(b1lvs4M6rNmHD~Y@Uc%yb&<}{kYw{L$(l9yDgr7} zXg}E~T=?B}#B$3XsY-a`>S}x+nbQJRa`gYGqJ^f}M%L!*8;3dOT^0P+f@1hga#{)| zwYVYl>F|Dl@&b^4r;7L=k(c#R7>5(bT-l$Q3VB z0_M7kdO&CzrCX;R%~P(GUXLc8BEkuZ;`$d*0eSuPd7V%LP4ftMOG>d$TB8?D?{#9C zcl|^6gPN_>;Uc}Ea0|Ohfq0^Vd+pbpneycAAYh$Hp`6hyX?tLhvq3LeAKwm<`z2BV zvd%S6&Cz!|%DeGZj2ooVoDC~xLVHBFP$j1RsuU(=5~*qd6IE~9bI;RoRCqwTs1w#E zQ<15i*WEwg_>HyuSa#f^uhjdO>Sgv*pCtFC@4BfFj3R zVyWq@4z$7~YiZ@0n3;el4i~2=2$xXdaqPA(gVho@2s}9 z1Tc^Mb()qlxi>5R2aUu~Eo5@AnKMYeSx{|=OuZwPL*_wu#MfzaFE2(sQuhC3G!vn| zR%9=*D7`CkEQnR-BXD_6%(7Ho_B?6PFeH}`5}o8S%~b`$b-L3Az}nH#ZLx5zV8U} zS$f!VcL1gP*#pcQY>)-{7};u}_cUUs(#mY0eV@G3Qta^%)wU4;5jwFXen3(gfQ7}C zvaDhMO^7Q%!p;lO3d0{nY{L`W1m>5ifwe~QnJO>cx@do2WKshM`*^N6$9Yho3S4nL z1dRx+Yf0t=j!$eY*M=4swB4jP>nbT4Qfs5pblGa`_|Li(; zn(D)A$ao6M25HHjRl7EAz9S~wkV+3D%K$aql1ypi-omH*nhGpa1072|kfRlA-9FxK z62Yu8e*%>Ge*n(Ri0%6mxgoqKeOhZ}mfQE}d;fy|I{}IuOD4co((M|Th<(96u_dOADFu?XeP|;xJr8t>Sd7{YNIPGxX3&RBo*D9klF+dWXc;mfeW&(T zner(6;P|Xph(WUjsZDc=gze`|=q6G341cL4tTY_FZsw)-k#AkfA#u7&4Jz$sw~Bc0 ztdC!jAt?z_Vk3^6KlX#vGET7U#GPi?&NET6YVD& zP4g*J0983~V;_0>{>;bJc^MwPG=6A4C|wnZDnhcA#eC1n$d#4MJO1guybDB8Vr|>g z5xwZf>qDNS(;zXZ%6`dizG)JyVF@w6gc3{jQNZQmmFy) zDlq{ie%E|u%h%pn)1B6CQnzciS?GFt#|UdIL??RL7f^ueMZnFnxHq44d4J@%Hi(Qf z9X!P|nG-vl-lZ?fo%^y~(`Z4KCHiCW7`vm2xsBTa#prr_axaJq-NlriY5?c}4}%U_ zjOH&I=8(KUx%a=-zvY;+vjWl-i-%r34>~S zS^!~g(qBL`*+9?InV!Pv^7)Zb!jQWyKvyJvq%(+4dmhBpCvYOgg`BqKy}?fmwBSXq7D44D8g*2)C$mru|iEGyilZuPymb>8uCv#Rvay zy)EJ#VX2VN-fOxA+Q~dTKEeR)XRSZ|{!fnv&~{$4;bmtZW+RCh2+R3C9jB8$<^H7n z&747ApePd2oD6fAM%kET%V{rk0Oh4OHFi<%BuyQ-Hp{sv2kL`Cod)PqFG$tA#s@3} zoT7XIpy15V{6U}DZjF$!@o%h${2&Xb*)Thwq5Gus3x20#f8jR>?@0dpWmoir3C_2Y zf}=#w7{18xOqvbXLi{R1h9)W{7iUw6#OperDyhQE~yl`vF6FnF9>W;&Gyli8GZQG+Ia_NHqq8L&eeN z#-@%t3;msEpKD{fGwJL0if6|&FT4RI@&1Hq-e@1O?3&#EH~+XJac2=JoO-2F?;oDrHgH9s^DSyr!|`~ z6+}o!=?<<5TYTNYiI*b=cv-!L$sv4o_msl2;=L>1Etj%#9lqh1?YVyHaT+D93e%{- z6ob{iIKJgX`|Low`|bz#iyy}6fl;qTQeiTJIYLZ+zETo&+12H_aX?i!0&O^xAVk-A z96&BIqKXz7xHl}+X0i)(-0BMN{OrB{MS>YT;gwuCv10O^hdUgxQn_XW z4_wgk4*|eUY|k%{-$etvbNM3Ttt>kWR`Wq$T{`ZQ+amb2c)mY}h4lHn$tLwwPB)xW zH<^EWuJIu0LZFV6bcaz%mAlZ&bQYyf9!T$n8;w-pJmv_KcnRk(&l$CA8&0xZ-e@`F zD|~sJj+1k4dXr@z<%0}8y~ZtcewhW7Y}bY*apJYlTJu}S5g)p~3?0$`ao&ypb>35y zhRO&-Zh=Y?*J^&1l3$D|z=JYvDM{OwU4p`-#b(1zD3jf<|tZ zvbS8FFgndmniqm5_tGaS2ne*Drp1snBng0k7u&~)amARgC7Coh=k)(8C6VJKBTr^r&kZgSsmQrA^5uduK8vnWm9d58I$ z)JWtin(kIK-6cD15F8A~pEIlpKA`v(xrQ&Tm3K4V8TWb?F9XaWk7V7%Qj9)#;;~I% zs6UlI!m@->4yo2)75aWBY)e^8fDZD9=e+)YwX0x$+F;%w_1D;K*Iy1^Y*$N*?TNqd z@4tWfqaB1;aIEzdoKZ3WPYYLTEr;ROF0E`o?dtR5l7Qy4ShRU-hO<1vE%x$JdtGn& zdOd{LUWy3=z4o}yE7lwzwS{hY_9cWScj0I% zpTYa|y>=L2i~XYb&n3rfuP%ZrrW9;sE*lt&j1@#Yu*O{0A2xK09eTkhZEx3jY7B|t zklOsQu@6FO^~p@mf_yz@v_JhV;kl`pdA_B_<4s>nZ&ymb=9FDK!GM*KmE^nz7Wmp^!>Us5xJW za=0IOf}=9UH{E9)44mth`~_OCoHeOw-}3@6LZ5%r2K*I?H1aH1ESfDp{4cf?Qs}`I zs%o9j$4>XFzLsqiMtQLMgm))x8+u;oRqclxi6e>9bmdQ`&uc}89-xUwBjbHI73@>(Clc7@ zP=W3lseCUF3YM9MpGEpO=$5=$tYUQQ5h;Ixx|Zm?5oteIl+Uj6Cig&b*-pJFcBU&#|5|#1HQ2k~iZ9k^n_fA&EupX3RRXB>u8ll$fu^)Ty~j{70`VXjzNV$8ndTs;`| zjQYraO8z}8Md2f?JAo2`u5!_5E51FSfw?5YqZe(ny6R-oTu{9T%vzp!4y^i;=^gd$(jqlHMf7c; z5L!oxPOhKXRg`=OYaGEZ&M)1(@AI#*pTBC)yPGGk#$lY(67e6GpfMPByNl-~fjekb z3ao5WjhtpzmL-n#xxf2)LVKWwrQFT;axI1I4`c-~RTEwYAMJ7YvrZc>-t}2k9yYC) zTVJ-GkYYPwq2A4bP#d>ak^xFRv$Ki}-vO4a+Si))R7`F%t5&krb<#Rk#=*rdz=Y45 zAUTn0Dkll$Duvn2?3K&5py8CADL`)iKi1565{Q#8OWr^o4v(*-`G+ZBGEp)7!1$3=;RP z1d|_L@>%FGs*W*Z1Iotl67{iuQSR4R;RlK&u;7z_%Kyxo*I~ghk>rR)+K zZ_sTltJL>&t>EOpmXFX&j%mt4t(K-hS}nU_CnP)WiN5(WU1Go`B}l}C*Z#!m0LHY0 z;u2u?PS8gzP3401kzDOhOL)-@0Pb(!nd$f)#GX)euxL?j13W~{Z_?uI+Ua6!xm!Au zZ$qBd7F~R$imV@Nxgsm2rz`d%ft{ae#DC56`LpZIjLbZKpS%O;xnsJf#i5UUAO*G? zi{@CS#bY`D17kSv-c$0wz(2`Hh({+3j~eXEgFsXIn!7KA-GiK;yoj&5yqEi`d3mC& zn4My|!h9f-{^q^&F=N8R`Kws_gpNRpzsF4flrQF_q)QNg-`g_;f)FBX)#Hj z$Lcr@nmu6d?BlPZHq0zmR`42$4Cy+O3K&Ynk^k)`^*=m{SjexoTcfg0@D-2>{+H8X z9#&EX>_R`t+J)}*&o1<{Pk!z~=Zz3XMB+US^f!DPFU3m>^mQhi%3#c=Jim)J;gE98mM?*8EDQyA727AD#@$C8-rEvDoUV+z- zBvt^Qm&d}cD$01GxtcO9!BlcX#!}BiM_VtXNlu*cX^&|vp;hT5Vtfwg4fF8;5bJyU zcS8ikAJ&%lk_r!YI*Gf zz$)>?YJ7yS7-cZ~UZ>Fp&-_p^* zS_cutr=&zm-GQ9Ua%})=9Rm=uqE}W>Oy!%44pyY%cD(RQANlf3)BJ%QPp5^ee2|zN z7=oS-2SUa^E>aXaMANjycba5x{&}uvUmX90F5&_jLH0}l45)|E5fTmu0%5G|^i3dj zH}b8df#(U9iu4uNPgmD}#{SBM4n#favIA<;qGxYb<)0}pE^2nc8vrm@kcPN1{P3dI zTcnr;uiePrH+WNoCH{D{R)yDGl zV6iX6_L}K|t?TyBJjRdoHaqj)e>{1))En{9UM{Wa6z9LNn3r-Ofrm-v3|f;Fq&tS^ zYY?FGrtrFTb+$lF#jg$_{3d2mvijQh&<8^8Q;&`}wu?-k2P-Sh!1I}v^JqUOW06l; ztg~!Di_plzw8yLb<>c>Y5|NVs%;hNQq+9Wx=!$Laf^2HyB3vP;sH4YvG}N97bwVb zPgvlsjN3**oy#XyAsa+@aVO9DJXuM0ZX!~un3>q%$_>vyj`DmHr(ZV*&1ep=)!s6- z;~)RqpSr??>s9%D7A8kI%HwKU922z6b<#b;d!8`BvTfmWmq*JwV13yGVdLC41tDS_ zD`U?G9*!e*L@VO{tpX*<4erT-%_5sUQ-w(p)}&Yuh#kX6@Bi@ z4XK01Xf*!a*#d2>5Ln%5Nm-5$x6f{9v(8>`kTzJdVa zHS!lX1Ue$gvJ$z=`;Tq&PgotLI>H&M?Xu12WJT%qVC>u3R=Ao(j6t?bM5I_sP>m;v z@Y-hV%KJd1tUWY2wVWQ;!$41VhgrZCHx7nZ!#;Yg^mbmA(Zn}5zy%Y$MDi~cyx7x&6>E6u- z>!5+{vwBLzk4NeMkF)tt)>lZ^#REFlJI`?sVI@yYb5?o0ljMaH!;xu=Fsu~J2_D6D z7C&oyB4^g$E_SunLVfiK=Y;tlL#xc%845rgvG~?Zj>}{U=MuEMnCtcok}x}Y0$P3P zW36Vubo)F(ExF$Fy=_|5DJC|;HD};@r+|me!*R#Xdt5p1Dc0D-eWUNMD*#9OW0jWo zkx>mv(m)VzA!dTVWaeunuz-J2zSCd%^pft(H zutQ>mb32G^`H@_F8_%bvfmC7Z+$L!lNldQbKMt{$?VZmI! zN9$&rXXdVMD54gFBb#Ja*)zu~$JP_5!MLKeFL4`J6*r>v0`hS{u7qBDgbHT~Vq)Q> zv`Z6LfW#82ooMwAjK|Z{oKKkq#x)fWKdWyhQqy8$?7y}igmAH1Sbc*BKkxDuw%;75 z1rHD(CB}Sig%@;DLyy{O?|lOngTN$eufQ5y9qmek{ zk(rN}Qakx!j|pt2!Fz%^2#y`sE)LYJ9(bMZ&Tf(9ocY#_t=v*yL$W7spIZqFlExG6Ul_XC%oern^FK;l#*iPGI!ydXtqPz{`Dp@7?_QTQ zb(&;I1t|juHE;_3v&4v#SOU%en;dqT)YD6Di#u(1b;U6$VyvYtJY=13CY+H$g2k`T z7w8r2n%}5l&=%M93f6ewXi88Qck3JF6+N;1PSUK=YbJ# zsvsTF9f?PbFl$LlVBtv-{?0HgOA?o_dt!Vw;nkTwMd#04fajx$ z0L)Cx2fuXrnZL2L5Q5dUyYa8Sg@mgL+u&;HYAhF`Z)M$)|I}!Ewn(l&J(8hr{>Fwg zpQ}sS`N|J}R{i6W(!sMb)W-7Rmw*B4SLB#ID^AS|Ne#n&aBS=O2x=~h_uKapn~fHF z$XExD_^$V8MW-fn@Y01u^FF1nLlN4?SuZfjVt8nNO?M>+v491OK^=q*VK2 zAt>DH%mp`MxI@9M*hZO_s@d#}E8-hmi!f;lVYDjGi1DekeRJ`MJk8C{kN#%)@~zr< z@T)`RmaM^x9bpeIicLgz=IE>GDheFFar}EBfD6N7ZqM?Y@Vetg)jtpM~QAXI1q zHqIv0Ga;BBU?H%eo3U#0n4g9lmGBC)gXR;u^_eS8 zXb~pSO)yMLwgZX>8#PseaUYWjwIiynNpY&%qDy6TLLct62mhO^vix3@3L*t zcXKGW7AjfjMe85OeJL1?sKM1Q{9qE?R8Sa!HLng~^#aG5el&sT!)uU+{x5>+Njg| z8`Mo58sBX{XE?iwEoS=zV`|b%UGXC{>LZc zSFW49uUrehP&0HjHuwT#kW77$vOHf;tB5c>r=})ukm+~^&$-hAUKMc>Caqv0klzl~ z%bK+iz`jur_FUC;Cvpl!8cjH@f%G+0kCP{&rLUwSRFm8ENSC0r5B1vpZO+K61Tc4A z)JR0tF1z=9>EA4#ucM!on&}N*6Ho~JOz!9i)thHoz!m!l%9#PT2gJ945FyC-<0Yva z`?=xjpFR(mHh5+q=?F`|V!HApTWEFhy=M&3=4cyJP4%6Pvm5oDL5pPhP7CPFQgDF! zP;lqx57l&ee)QRmDUfEr5geod!6CWyeIVaW-~4-B5~8bfV<&*{=T6gVlSgs2eY7P0 zi7(QWAk1i2Ro0EWrt9AgEA^5mzC%oKf>PU!!&usnqB~=%9|%!@d^*XMe?Oh!|M+zO8W_BA zbWzwOH7m~;h4>KWn4%|q)@1#n>qZCd!OV8P3#7>EdBI^t_W^d8n1YVwIECLVmnrPoFvmbdy^`L+!=1OOd@DYI5&e3d^?7c$?6rpep0D!nsVv@TQNqx|B)(A@kLUnDv+tqac<*n02WZt!DSyA9 z=fe2bKMf8$xU7HVBa=Q--PMQumz1MJ#_l_#wK-pYUJ=~2B(dKqG|Tk3?V0`01%2=m zL%O+?M*^|6*O`lJo0z5$y}PCo1+(+oNKQyAbKwuMruiH{-mUPPZt+EwIyV4n*e69q zEKIkq`E4%zP2($Q9HDMf^H3X<=H|t&=JlxCZY${FObPh=l4YZioAz2vDW2PugY)iu zz=IGhBDm7`{j%m=+=+|6^sOX#e(1NV>KAt9brHR|=H+-GGIR)@kGPj<{a*6GPULPV z+}_glf}p{S1do<&GV-f;UqYN^E_QuUxsmd=(hY!4Xk#UGqdW7m{p~l8F}?l-mQKhsLnd(_s|b92 zYp>%!;qc1;>(ZSnRe%P6YF3hewkc20!KFZSK!q@pO|iv3mOxO?ConpVcw@@YL}tv| z_(M?0uCfudg#sWYXh5HrPru#Q>5X#5SR z!>PHN;<~zlV`u2yqg=FHk|>jgSF)(3ixTJX3iojmw9ldmMhYn8ov;0c?8;tM%Eh`8 zKm-;#Qffj!_RbgW!wlJX#db}UL z8bmGAuI(nlJyS=y;K3m5D!*3SMH^Vyg$}=PB?%dTKL7+Z*&+CHljj zB5Ud033BYBN=vi3TiLCUo^7aA;*2UwM_o(<*i#0h$rq^jcv)jfm60j#pXHYE_W$P_ zL^2c==d(jl62Y%tTMAqmR)N^PhT>-Zr{xa9tf`E?#5{O1;%jjNzaID5H>2a}x6Q2K zM@*v8(?FlBwps`K2gxAxYBuJx-I%vZ78V5EQiEm}V#{k?y78U7pT z|23?Ln0Q!PLe19j0Kf*de(8q{s;mSWzjD`nizT>RQ~DX8Pn*ebL$`Zp zSh`-tjiHC^AK&ZZMO#c{49W(-Y6Hy^A$W!ZD02$S$&-48?4Wf@S`7t8ZFhcsBaqcz z73AS>wVcY+-;l;%*_jcO(;6H>3qTf54)z^+E9A=Q&9!e%f|v-gQ!Rhdn#&1BPHgAP z2Y??6s6Tm>Q7mJO{9(qh0wQqpGw_q-TWO69ay@Xht{SeL2wXn@BM)YC{=7CHIQqWvs1W2`;lf`${X+PjP`wRf~tF~$ucBD3pcB`DL|E%oF2UD>c?cgOH za7bNW*>t%=J}?tnEmk=;B*TD?U;5;%+c!tHe{_v^Tl7zY>(VA4W4Ppgdou4>7VfiUMSv;vAwen%##>$IF&J9vXN(1E}5RL?p5LD+@)ji|i!$%mP2PF8N$djH$ z?u8oE;S;JpNn%$1W;vfm?E3v^AIrOoq=O5&rSs$MYa6}$^iLNs6gO{dHm6Ehzj7t$ zo~)TR-E?vvxCK!T#cAhQrvntbK)X620YZRiwTlK|m9LcnvcS#OxZC-|1N1Ihx!=O_ z2COS6=mu-k?8ex&A5Q|@y?*W+e%xTp%*Ps;oH~4zq=Gak)6MA)Q#85^Y&76q#>3#G zrIc9?45z;S24wPW@dQ6$%kCcs8_9vST%IH-ec03@XrH_^32x>=Z7$_kP$MVUvml)+ z(FCGUhkyTF797DiUp|sXe+{0AuuQ$hU>Ov4;8LdOZDqvkyY&Lf(qA@Lh{w2D&N$tC z-*tCVe*$&Y_dI@C2rM0yi!tP0(%ylDi~?1K4>(a-)(r7~o~TAnIzD(M71Hh~GM>0k zrRzt9?Geqh);ApN(e+AC3Ho)!0Lk8ijb{Sd{B=C5zZ_FO(HfjTxgLkja1-pJ-^ck^O3i?I$a-{{28S}sHm6#7v*i^*C-eR0 z8vx`q4x>_E08geM;6Dev3nExfEdt-dXrGZLSaS>R3|7ggF9jmLJq$+R@M58DTYr!5 z@oKV2Cg2I){XtV3L4p!z7wTldXOv#X`%Np$JD#7KBQ24h)2*2yBVBQAdPcQLZ2`qZ zICC2X3AfWV!}uQ|1fql}Ox6p0wi#~4xfU_Jggz}+*FO{PTd`lcd*~*xP^0L2y8=_K z?=LV>Do;w^E-{)2sVpVjc-$tIR`>XxEDNRI{`>R$c3HrESNVQRwj9ERf$ebpnH@e> z0|SPBE>;1JGj7h8N=sOdd%M12TfK>Z-^_(JaDt(Ph+~YH3Ui3@32vII!|`d(VgQ69F!L(Za)s zCre8;Z-E>UsU$;At zgK*>rzSIe@j&$x#@b?()7x??sNK4}6eG0OO4gC1a*lSNL<_)`-%eX-70T!e?fBOm= zs(8cvi1XJk=bv}oPs9q>XS74?xT{^pVIz^y%z||DpfrLj-q#wIU~m1P@>m<~B_hJ9 zWMD>WUxF~nd=hM5BIA`sVl}K{uN{>hww_vwxB6ID<4GKoAHKXu|G0JG$L4w!$biiL zwVY1iL+|Zg#9&o^bU}s-S8(zsXVy2n1(iNST>;zzaDPu5)<&w;00X<6=m62T;@j)` zuL+5`NsEVK$0DiECtp|c_F)X(FPHKB-#?1`c}p~KwsZ+i-IpAp(;6A+s4Rneh-`m- zaX?EUW)YU!H`_4Q7%U(mjzWq{*?|dNMFPsM`SPelD`vn6Z-04iPARu>s2TYEo~37B z98II=@s)=_{bInr(=&={^~Ea+x62%zm-1S3XirEJUPA51%NvvCuLh^Wl1QTX$%&L% z*>-8=La{k-C4r@nTQ%=}bC}$6SHI5h;My9-)hD5@S}GSf2LXkw&#tAa?x9)boj*XC zq{QTV5;+S>KTjeB;w7%{bnNw^DMS6iK+hh}x4u1IPv-n<&WW2Z_Iw2P-hcdt-GuPl zq(0X^n~asVc*s>(fe?ly9YX6K$}bgp^iu9JtjBc>y>Ce1h3>89mCmRYk>??4St-)i zE6%THQV8d)xV`pg7X+HOxN<*@I%Kc==Dq8i{??3=QzPANJCMpeGRZz${xQhk^31#9 z0C1HKcyk|kn|eO2o^dE182n%;^Dbn+zTr29fQXPfI~;*hE9`L3zaRZqUw2GYh2Ps_ zS5X@Jji;_=6=B}J5EBp8j_}>Qa?8St_r7P31jWgmZoYl|6i$FGt?4AoBmYziyK28k2&f3B2*5efs;QNp=aVhzpt}+pjXF~V;ts-tUS<@1!4J~cr zyN5bE+XQsMBHx~IYcJcXI9%(s!KymT?nx~Y`?y?5ZLk1qG&3P@HL}R|-sC&#uw}Pu zA(m~{xrh%~FIAfWTbU}EM^8LEn_TIWtHxWtn@uJ1j5}uG3j+0Wx+i3eF00kJ0A;tf zzpr-kJ%im6t9>;#k@r0hNbKyVm$wsYd8rp)C`bX`lY^6)wg&S%v3a5b9+!JJsb+}YuG`n^Za!mg=(KNi7k@2afa zpaNfKtRDa`E@~%^vboeP#WBxs;eW_dCgNf}r45p(-($|Z-N`ybH8nP2rwMNd1t(OV z?32%g6kr5m`!3GZ>OAgyFJ+JM7ad7GBXwE!W6jKiVpo0Mm0(^_^ql@Z=C(cBqqX0t zzhE-HT_1j;Df=Sqt+6>^5}to|Y9jJj7>50*UJm6U^3*6!HTfEydPes)DNwa7+P*5z zd9MieR(5pw${2ff28P2nJG#h)#_q|?*fr(+<7ZtmDB2bD64w5*NUo~4lI1VE^sTF( zVfeTdZ`!RY3sY%DTfZaz5MkT^sK^kRzueK#_D>wXWw4WA)t-TBr5AGN=$KRW;o1qn z?YLGYpn3XvFoyCc_!n4K3f}!&3Qo{WiNGcP{9R96rT4$VpT*qQ_RdG>9)+K4ud-|c z+rgI%nDvi{(!;e<1G_qE|LhD9#~!(+w9?{=rXZc2COA#3X)|X~!-%_0KNdLMO<1~c zb;AU}pF%_5mn!)D*}5AbFu@xeOU}yUCR@MT5$WGN_qVEPCP zM+e&U&#C$cj=N+>gJhq2b@CE?6U?*eP+WcnnPcl zgQf1qf1fGd8<^!R#$CqR2!FeSmlD$=3}^aW>C0dT{c&Oo=B%dyNFX&ZP;Tt~z1+}H zP0Lfg{DAAZ_SKt5>nt&Le_z}SSAdSDl)UkjrtCUw_l`JT+o`2A=QHqfumkZ9?$%KX z9OTn-E&oEt1+QqjrJAgI2>P=*pJd>{E-|bD2BwOg`aR&VuYy9 zU_(|Ku{46%N4}Fwl?OILGq}Ts20xfp7nWBlRkBdRYU%XB0YjM}2KJeKk+)m0@y}n` zeu_)|X=wD@n~x8Rrqy}8QbnAok9`Nm_gl6=_Wf3hii}@alEhke=c0l%h z%L`}z;yO24;Q6@p<6eo^a3h(6UA?g!tqmM%N&=eo8Q0_3*vF9{T~${CS>Xfh2Up$w z%=VXgi*kfR`HjbY?UM1e`;86rQR>#c2cXI0k+Bz!oHeW3j-1FINhO40;k~7>F?q2~ zBtuYePp)Ed?ERu&Q^jK6iX>vKT!t8>DBVTsfFm z?>@A)7Ob?X=Qkub*wNE8+ogK<|g20^xrD^xCpcQCdBJ)1$2Wt z>j0)R5TbIsNESahYqD11B9g(?P4Mb6NN? zK7Qi#+7TV|)8_oH<&T^CQ#5jG^v*mCiJw|3;v#Cyn9S9f@y(?;&Pl#8P?a5A+quQz z%29(OSFSf*In00ea55w6djC2Ng85T3GJW0W13lJ?^XWt7{B_<7`SEZWyS#@tyT~zf ztLm+fOX2lLP{T5HsEs-Y<}1*N2ImIj!Rx%Wfw_W zbXsjK@M;LzTz73>-l&`MZ*^zSFdF=L#*ya}v9n~UdtcV&3ist#@)MPDvH_jZrE%aa zC#KS;VKKJz(a_hZa`e+9FZdSv2d80rJKEhX#a;2Qmr6DW!xdMt@#>z`p|yu0wEn8s zB}-IUY6#t(&u`=J;z{2nH$K0DnxH`;p{&HCe$mR_pPqWi@ZO>GZPcWWX#ZK`5xi}S z9DsBm%5DN5st7rB(p<$xmZk)r2pWiI&>9zii zns+AyJxb&^(sO8@CGDlx`>Sh8Gjf}PD@Nnt&BD{@4yktOa~^9*Ul-dzQhw^0dVgwE znvv8X=Z2EwhlvSWWX?Iq{W|?M5}9e`OG?YGITU74vF*R7hes|~3E+vV)3j{8)}L%Oj3>7hEy6D)QXvd*wfJG-X&N z+Ny6fic}pm*!Y%blx@+CFQhEQ@hYM9!AM7zoTS(d>7F49^B8- z3H-J5vyNT2)1W;LiV(mY!o*jZ8XF$vG^yb*cD#pbz*HuC1e&Gg{-IXfsG1hut83Ku z)iZmJ|AVZ9>j*YR-ZH)hwttI*#P63EFP0wvj*7X9Z^Rr^YIDZu@#mtN6*&+(y~ej2 z1aeE1#X?+Unh5UP)k6-Sk%UHItB2ZXYNMJm1p7&RS9|kfa_D*#j-l$@6&yt(bC^u` z@X|iRWUgFWt0^U7BaBLd3VqjNx!YZDW25H%1{(r3yk7WZy6q3qmJwvsNjFt^%;gNq zY24_Ndxg%hnk(c`39Yf|=h|fMMMPJds0eY@4CJnwt@xN`l7g|*mF>#4MUzjx_%6&} za@V3ReZ9Tvx4qAF)pSs4ekF9ElR_B~pQn@OMs@l6EiM{f>T7mxkUhGdIHIxA=svkT zv{s$zr!!36(+o%Nl05f6&-LxGF^#P#WnXja+?!JzGWqA^hW(?{d@!OPWOo;5*wsWr z!o67W|6ZpzpplqC6g*%KSwpDW()=T+WgXs>!bMy&!zs3EX7VQgQNZL^;JuQyiw;OLAa z%wA(~i5SW3IwIqC!OQ0ZwzuCgtmu*9ep1ucEvbjDB|bP5U!t$N=g*dL=9zfukP3lF zU}_UwK`T9yFMFwSJ%V zs^77t#W~JqwAk{rWdeAc)r~bdUv6z;s`z;NjW1E(ckbC7UyR}lNX|I;c;El}yNe7o zKEQhCZklnXJiV#H8RK$$K+tvmgY=P(zm6M3inO-7#(`MoVEx-e{E!MR(%sb$KjJjr zop0^7S3V$CfOM#bKBkW&OgVcpz!i9b_fkheMRIua+G#5Pt5%QhWtNr|RtJYLIOomBT_z zgPUW^`*Xj==Knf-2lvfagwBkmrsgz7S9~V8MHfBQHJkr53}#pfAJdEFxpTq6%YUhx zF|?xRwXqn&N!&OH>kk&dA5VL?(3N*o=gj6MF!4#ffAbWQf6LhLWyvjfk6%8_fBM0g z@KJ+5I6V-}k8(TsiX*^@7VyojJLxrUIq@#BgxOYNDLxn7=Vc!~q-2lfYR$Xo*;!GF zp3qHBKfI9$XY_-Vr?<;hkCouK%h7Q`cgPi`C>Qt5Et{j;%yv%7Y*8_KzxQ1Eb_Y>U z6OD+$dxr78>KdkEn+LPEZmHNE3E6Y{oYk{OM_xT@U%PhBbpyZ@(*G1POH0)ab7^xS ztIl2n07I1}wvsOf^nZkx+ebITFgh;-TW66N|L61S+Clf}+B(trzY<&QhF zwX%AXfJ%doM_ZwWm6oczJmG67=0rpLFe6GUx#?l6SI+ofKU4ADi6j3XW#0kSRMxc} zotJS0l|d;Mic$mw#7c))K$?Ic(jh9+gb+jTv7rJ&q^p2Yil|5s483F&A<`uXNCHF% z5Hy5PB$R~YKY%muI5XdT-=DQ+tywO;_nfoav!A{9xxpSyChH-dd;X@2AZdcHIRMB# z$3HD+@!dfDA=3&0IS6k1zZ?SOy~-bsb>oN37mB$lgz%pr4x9ewY(P`rt~I$qMnwN~ z>@G8&2*ta1m7b;a$_ZPnKPHfKC+XJ{Ga|q>-L0sB=@Z8UA&|>!HV(PkKgKo4({_XYm=$>@zPeLVJzE*4=Ci=Z#vmcvUDhlb0{M7K8&5i`T9QZ|wK5h;5>_=3 zsF4-6QcFJ-fyglk<1*X!E+Lc!vzZ-NW+*%pE<5$V-WW7DUKQ=_>DJ5#=jHeIY3b!R zTlU<%_p|X7<;!fj&w*%cy9x_B1UbO@`(d@@J(SKJcV3r)z4yFfkqOTcwp{Yx+Bxc; zM38Py)E5F13i0M3Z}|Nkp%nlAghMfWVa8hy~=%$EwqDwJlwb+6kcXOPkaLIC7(rx)pB{?t>}GbUGOZkxT3?Q2gjmU35LC?NUqI z8wp@zTAa_x9=D-+p-yC1V^UyeBuI!{#PPLc6dcoU*Dap&@{$iKgImVweJy!*Xla4) zX;(BqSkc54{gQ@MBaF=dhd6M2klLW*1AFaJ2$aT$DU?o@IY z<0dZVwQK%?35k{)h|I>^9uIomu1(LRCY&oIcaoN3$iCSBdR^e3rqbU2lqH3rPxcFm ze)VxE5M};);+Gu*5u?wspFIdFuu$Y&^#rQ!p9LEy3_VvbC4&u7bf_6EM=L#xRU&S6 zkW#nl4?%v_Eyqp2*wIxQ!u6%l)cZhsO4tbr??02l+6(mWhJES(@Wub z9BQsauu!Dx>nXBn(wKN;5w488R1vyFBpH_PNG%%_i$q0mHM}tCCS$@91&_!=-0=O3 z4F3NHNT5bfW~II^pra?-3FInm3o9dl3*u7+BrAJyfrZQNLv=^-mwB%@Lu6$BZoFR; zbFVAlozqU_=U!AD60^;Vr!^)mdie)XFYPeRZEJ%YCY3&Gk+M)~-4>Pcl0C6i{V7$- zBBleSTV+lyZ>!>Y?dy{Q|DM5Sly(@VQ1rCwT3c%8OY{VIpS;NyVF>YZU1Opi+ZBkj zFwT#8o7}5_dO5xn=4<$#Zck{D_%AvD8#zIPJFyS~&RH;(fIx8#-ca~QX-N=Bb5gJg z+%hw7s5dVe>_h(V`?wqTf{a( zbd+QQLv8UDHzox&ToEo^WXl*2BcLsHY-&OycaitQIK2@u5n?o{uX&-%q4wVGUr)Th)37{W@uAdx=v};bR4a3)%=Q||^yzYF7i4nH>wsMQ~nA$RyWMF7@= zZQa56g}1$VyIJ%fSRX`^t6;9Q_bzZb5kqeYOk07l6@xwh_Btz}ew$h$`+Rsfn>>Mf zvF`lH-?2)%sg+QY4JRdUyh+tB>Ahc?J{t`1IXILEyLpkDk~}f8hc-Mg#lf70$1?Nc z&l;1Dq$Z3qJ#=G9moie@-ykq7?gIR_D_A&Ybh&qI-}15=k~lVs-lonRFUwAxvrkBb znx{p^?xhFmlcH!le^Bxp5y<*1{D+>>Amo#3>bsj8n34`3PX4mzlD;3ee*QmD?ZM0s z-Ie1~l1}q+ENBLZbU9O@B#KLt%|uFGh_Z^qo-`w+M!y)P=F2xL4YmdAyv8B- zd1X8$7JqgAXv08$OMZv2Em}@SzF73}!sLx}3DaaR>Vm5*!=s0)!WeEe&WL+P2+~iR zm-*47Y|s8sJNEKKup#Z;?H)nJuK+_G;%t``CNAkDHh>#{2wY%L?}*qwNw}$*NCdO#>b|WSHc%sd()#F(0Fg-Cm?q6Wr3& z%;n%+)>>(J=A6Ri*>V^>X>6#xfWU3q&b=R%Tg2lEEPg3M&j42KUFj?>7k*NPJU4%I zKk1+;SstcB7K(!HD0u!uWO{r#{Ct}xfJeFxNI`&@hT7iU&bw3V>UiNS!J3V6gKU@h zC#Md)p^|XiVWNTgA$9y6rJeZgK%qqk%6w$7kk)B3p|ov9qkF1gTstqa$ehm85?vI# z`_z~)0bz??o+ZS2Q)i*7B%S5CBw3{YEa80#+L9=*B?#?2*u_#b&m#+YF%7Z*B{+aJ zKECW&K1n0MziOyQyqq5^J;@&dRC<%e$d;5h9mzA>nXLTux-PEA4#+9Pe=tXDtD7TY zs&aD@ZPN0E3N4tE0JD&1%Tx2NDw7PAgtys~H0Jt?Ov;qLRx-xJZ8xGMjU$)UN+)3f z8L0{Kc*Apkn>b}$JA1ar9=34S*>4)}+H$tqm&tIb`Gp-vfADS>%_3@M(}9at1>UVL z>W$3&7R7HsHLQ3}3kyHFM+6hl?ewo41Pe-l90>ad?afP)mR0ndVVmHxi#_*P+h8?J zqD~P~4-SOEGr%oVshr7=;l7=&Np8OO>`PwMcMQbV;M2enGzCTD876ubmYr0qFLs?= zB;gI>ESZ>K}8a|qg#J4!&^$P;n5=)mTIkB z>(`9B%dj2xCC=&_tP`jkM$o^fB`tYGX<58ov)n$1*0j7U`$+OyMY(}HQIDeb*woTQ zbtUZ=49+K2Xo5Xi9~Q*d;w`vz`vi;HSW-6oo*UV+)3so_x3>xf#K3Ycqs>o4Gt9qf zBoANMhMwPh z^In+y@QvV%&-h}{LLecCCgUb=(3G8*g4mezknPTE+TS+X)TS;d{5jr6Xmsv%#~qdR z-+G&z4q6kZPex8@>eqVx7bGj5m|+v*hs3muiLu?;j2zC}eelQ>s0;4bf=B*wq78Wx z9piRM6$}qA>MYrCUUT1n~a_P62vLanN4VaV+*`Ks`l={k6GROTzE?rMl~QfDyOg@C%>?I_R_9s0FtfJ zjV^>uQF_ltV~?W27RPQdcB$3WbMwU@r`E7CVlP&~Ph$w2BUKg&~81h zk3Q;=)C5oo`@IMU!dm5!1R1TU6ew+6uiIbyEn?e%&EyYm;a;DZy3ax%azH}L8w5Ko zWlmp>xymonfZ|lyrdD)xo0JQJRHfBxHsLT&=)bo0SbfuKhTry9RWikBkAu?XZ_*~V znY~6#@b!sLzkIdVw`btv+*Py16FQz~yX4Fe%>vjFMKI3CX3uN&yuKwoA2PhsH|QzhoE0OnL;o z%nf{q1AcH;W;xqGQGfpEK6ve3l3|h?CD#mxdDo`v%eCaP@+_=t%Ve_IU(t!-KPa;G zh*E(qo?e6PMmdd7R;U?K(lH^sBQ_PIplf=ICPwy;51*g96a+*D{%ELf#`xcpx1^i-vK$TUVE94n~=roZkEy;~U z5C=52a&#nXL`4OSk!~UB&3nFByU64`GFrENPj2jgcxiim_Ai&Ka+nu_2}7UfZeOR! zm%{CrW?4`S=+P~9cqw0-*_%#qIP z8Zj9~?@%fUxa2wR_S9C@=S9$R1w3pkT*^%|;jR+BR1e7>DUUPF?9_@%z2aJdT!uew z>q0289TbBjaPudV)SmX(y2R0Kzp#hOyDv$ol1{~`Z3u0X5WvHUnG(G{iIwg4%kH-`R9H$K+88|n?rNgF{eK{ljSPxF9ro z<9N+QliJ5gN33z#2TmA`y+_gZ>V=f|ht6CAh_>$SSSfF+VRJ!L1J`yNWMm4=_)xVllV-s5VcWGk*>gn(787Ow)R|Y8_dPC^b!h%^ZqQ=I- zWO;#sEY%InvrzOUsjQp9swr=&8<=}HKOlz7@Fi`NazO0Wi$H6lT7LF_lQaKNKnhQZ zSZs=a!~WDM#nGnPExp3|xUfs#~gicAJEOJIZKvXc+}>$uG3w8{3* z9v+A(rmMw0+Z&_#H|8gNP1|$o-5D#CVf&ORdux>YQ847JDEgXIEjglv(F> zjAu6A>7Bi54x2{M61EB>B%>js5z;@&x>@xVF(JNY3@tT)Kt=W+G`>tNiqm%IipyC} z-uI^O+kxNe+gcH2>_2n#?Q~1?9J>7trMDvT*C_1=!Mbv7ZwlkI^&Vax`_i%J#lz{F zpro8{Qi+t^iK-3CEQx6EUvi+u9{T-&MvwzD6djfR=tHz0XdQ?(U=!Hh(US=oH!|-p zd>FX=M(R0=-#N>WT&0Cj4k&!k*Hc>-dBT=FJ*SGOLCSMUm42@m{=*>yG)a)d6yvR4);eY(O)TfERqAs^Ua;?ui#eRxN7I_}uk} zdH3}Na!Y(28+*&B^MC_!Lcz6-{oiXv_@!l@H-!%ZIc57@eo@8gX-B*oPI*P^15ClV zA{=Ls*QS|3QGGtuM9KHmo)5E{PQ5}KV+|MrzcFO3~iL%dS zSfouaA3RlZr{p0jnakBD#3cvZeX+yBy2~*6h1Zy%JHR` zUH^0Vfb8zspym@oScvzovmXB%G7V75xj3GYM2T0SiWMWojdO#}dpv-{x7C3S+{La2 zZ77LzQ3`4w8%`uGH5eh`+2r(~Ifb3c$V(Fy%gNSnDe1Y3iw2&ZkwHbaTQT8gEX5dn zhb1>kzp^)Hw*3Xk_0bV|4Ai`5q2Iey9Ixy|si2NC5ov1O-0*u*hs=rWCvYgqg>-4T z)if8?AEn9V8k{@)=nCUCPK0RQXs)HVvwXjowlQ12FYYytvPj_kvi-WG>kZM?VE4Am^M?p%^`gIVX>=M~w78c{HKxL1Q2lCxs> z0=R5)aFM*ob-5-I1$xVJMY)OsqOh3Uhpvyx29fYnvkQ61<$cDag9YCxS1GnE13I(E zo6ItX@j{GM<&|zd=Mk zPei?8nwDa%5AH#0e^dKtPD8))Vi}H$&63^c#T%ffwiZbebl|mA(%rFXQ>zY?826y@ zQjHglZ!a{F`x`49; z;&jB6Z|Q-1)=3HK@;;NC4F9zKk=45I{Ho^49RF&uHKS=B66j_@NKQ=z*DFc9?h9d$Wg5-R6yy#N~s%B zrx%@Rb8Ec5R-@Bed%uy z^fyI(W)133qrghE_lJC|+Y^0VzvHHvcYNfgrIXZZE$Ah|l{&4`DiEZZl{V-_Stcs! zC7qDw1&Zy^ett|b{w`h2q>1TU7d35Fikn_utOCGzgFOu`Bw<>Xs83p4&WY@&Tl)Hv zv@mM;+)yuq{%}uaZ?Y#jE~m{xx=60A3pwk*oDKdGD5o8Am%CJrEyClu^zPWn9L^-G zvvv}3_I((N97XYTG<2CSr%D$QY6ZD!*)Miz$u z_1{}JRv!^Ry{nfK`lQ-FueMs+xcO%l=Z=|K3-z^6#%+ z20acDFAED?JsfRmUZGB<4#$jIqtOGZ&?4=kf?0gu$qkTW)W0^v{K<1XvmsXnLSf^R znSe#q(VBiTcQST^qb?w21eho6K^m)bp{q)eRR1&NnBHG%5a2_KE3F}`zmLQHY+HQ! zss}Hi#?O}b=ZJ1AyEGy+XrYw%h^2NcK9)I;< zjwL7-Q$b-tZJXMx9m4oad!|GrfRA#ph`{;|QVt*ZV(Gsv=p<;G;ETp4cyyrdS6iCj zt3LCy7v1GGYqca{<#}0f8@L8zjb!NJe4M~7E_JOBR^VXlJ{q2>=%&5G2 zvuh1gg3`UXCp-kqk^i;p;?L3v+p*FEvRcOfT+IGB{MP0@Xc##7=3t0d#Qeh;#KBwI zXVybz*6iVrGXC$tW#wQWKicc-G)n^<(FS>)iuTi=0Bv@}hvehnz0%7-s?sLm7e2p$ z72p03CoX_kN%yeeZ@CjSO zn-jecneTI28Q7m7_fUVSh5Q+L)ITq*rh3Djv$Q)XCC61xwEDlT_VIO*H%KWj4GiQw z8gPR^9-jQMG9mCM*!A0$hp)Wq5r9_VD!zYSwm1Cc{GuR5hA$-poQNJgSPXJ?>|d1X zK>yyLLN0aMIcwf%PhNkmrIJyrE0CliNv)83eZUO7-1J{w{_5~pAyvvQ@)RiMbEka4 zb+oyeQ=hdC?^&?_R57+oOs8VVcESZj2dD4RLAP?zy^EVyzG-Bs&*Ns%g-cBbVUb<7 zw`;_bmO z0z1cn9v;IB)UUjbNiZtWwpInE+2cGr@*eOv&YTFBlHINwb?u-*%5PJ|)!xZJAES{M zxnwAv-&q3cOaB_e;M`_f#tHAz1a4<@iFvGo6m#yW;b=p8NBo|fD6~f>%=NtOZ=gGZRIqsCfba6zld zwGwIwpzjc<^YyoaKRJV8y3p173a@5H;JHG!xbtM>y^GBn|ME~)YIgwzFom4Uk0$j) zhg|1Pg+@2i2JuVt3}4V>)W0S}7fbyA@?UjxMOtx8D4@ zPG%GY-TFnKrD_S`kEt?2OCXHk*YK&Wu%+6TdA&3GeRiO!>D1o=$SKn&-UDo)lxWc^ zEpzotKu;CV61)2x!AhN0dNVxoK_B((T1*M0#^w)iwS6xiZCkRR! zubhz~6UU@mk{tL?lpDUFAGE2Yn;Ytp(AI})Q+PuWw0b= z!eD7&wyWEY`oy5r$($?jtZ`PS-QR-s0f%<`s&84;Dy?=n=5grcXH>k4w`XK> zu45~#waV~e$w^yG%1INbKRp7pF+v1#*J6O*gm$6TO)vOY~X>os*AO+||k5HfhfMfQ%~Q|}6cmq+Ku>lVmqp4o6&-oGj1jrYWBVB=+*YbW*m%>%gHOmP$&P)Pb|1I3Ay<>?&u z*ut`NzeZ}p9(o=+*-a|iktrS7&sv`Tu>H@%-2K4jbhqb|JBN-6TkgFr-Mp#zESQD- z?SLj=d0|QcfrD$6TzBz8r-e_;Gq1flW7WKV^KFo;JxObMuu1G4C#0p5kX70VxzK@9 zW2IXrm(L1v_0CzzjxL?kDi4#w~6Ckvuttd z8q`Mlz&Q>ynhQF-BYA1bAD#U1RAZbY4WaJqE6rG_3^b<7wjK1!O73YaOn%q4F8<^i zUB8wDCwVnk$BV`)oINSz=BJR|`s}f_}53Hp$BpwBXj-V$n09_^o*W% z_TG={`U&!Q?z?URCxeq#NGydkmB#IAfkFHGgNFIZM~n^JjGZuH5|~_MN&CU)uUcVZ zy?a||(lPV`2^3g!Z4q(VELhJT!$Z-S1=_0-AWM z({?3YcC=kB1PxAEk}`T;@FmngI$sUY)VAYT00f89C%%F1S>3e^u80_$m&Bi$O>At| zGS{_!t%Bcf8e4~pyt6Zbnq2x>*_c?$^f!3kn({4UJ{=w5JE$ zjW8xO)w;BBh8snZbWWvwr1^5CA$idikV~b3mRRRJ_E=ZrGXmJbe)ty8O@EFT_f9K4 zy^K4f|6nI*i3Q@g^+^#BEa&gXQ(i5p^ABXrf_$1_m~+SP#H%)|bsjW06-4Tj+A0u; z!y5D2QGh#<_cHqQTUB==e_B${xszq`u9fSLs-*fVX{CL$AR+9Tq#pW$XM<+~zV-+< z3$KFO@|J-J$Z()MyNpI1WuGxFnV}_q8OwO5X6`7Y@Ajh|NV8VO!X^+0?RrKcg z6Bn4MlyKcDhC**U#YLB8DO-4w`e#n)>LrJR=W?Q@n?HiBh{q}%erLV|h(hM`b?h7u z0kX+RE2ABvNA@)&)F)nXbe34DqN|Ek+2A|OHvrRE>w1}OR<$PbH*w~?l49GyV4%rj zw4?EJcSo)sd&hHXvUP<`6RZCqk-w{-RXWWHz06j=tjSHCxB>d1WF@K_uc<2=c~?nB=qVj=*jlk<0V;+&8xgebymRiA%4 zEn8S{6DqG*wG;bRaYwJ}ugUu5N5u(-N;rQGBd?QG<%Yse9#zlB!58QSnp`g`+o?L& z)k+0)mN1=(>@k<3z$nrJ$=kfAQhu6n#}70CsCp%X6P9YC`8Z1`@35RabPT;23Qa+5WSF}m0@S(iY zK>B4BcR#Ap@JDaQ6D&F3jCy255`Qn~th@rx!{DX3EN7uiGL~XU?J*sejkj#ls|YeC z$s8*rZ^`~S?v z7(Ep^d(=T8`}Ahe{|pR#XDCH_&Ds8Fg#gb3hS-O(dW)Bi8mXJSIedpN(4bpIdZYg5x@F zT%jkk)SsKlvF`=7UQV2=ou%!9+ZHVvpaJ>#V#Hre-fNfryb@f}H|wq#0dpIA3YY_+ z8Sje!xOh>!qX6LizWA-$y1*QcTrJyRg8^Bs-JamskNmsoiJ%y5J9(lK$!R=_DainI zw)7e|kqY}Q#pX!*$3`i6?D^)xqSE_{aP6pdcD4NZ{r5>~17)&v8hS{2^h$?ZoUbiC zzclgPH2E^;AWn0MY`QJWv9_{UjucKqz_t!x!rD71w|L>=y{ER$SDFqZN+Sk0WZgOB8=&0{!acKDJfH-N7a3(4T=C=1Yb4=g0Zq< zS@5-~+KE--uUZoV`TTnfjC+#ZU|i?<*pYQd1>&lVmk9+cR=_+mga@jm~!> zUNK>kAfEqHj_7~Ui+KP-II|l&njw%2~ zaT}zVgq4W^mDrcE+oWn7C5Iww;I0JX^n4Lpq8j+-&))&`sMr=ck&un{QYuH1jHay| zu7R)3zQ(nkD5HzH10qbSpPuSd(6hX&iArS#J4oRRIoQNXHq~v9s{x_ikF)G*z3~eT z(P7|{9Oz5#zE{^7IZc}!o4|{nLoY5azn(EC2yyiR8KpMl%z0o@aeo;*qg^=4J)srR*)!CcU%x+2gp z5z(*(T#9w%d*XgzyFtpcl)s(sOfK+~fq;cSa}+(df9^|2;fDbvL(}~WnQ;#kM3TSL zY&R_RV@%enKaIdgt4tEQ-Y{#~uEt|5~_*A@b8$F;Tc zwv!az5=Cbj7n%=G*k@#U`HEAKV8V0p-p|{m1YFVnh6rY{|8=M<HDRZk=?VALLWcH6#$JIlH=dI$v`7AHXlb6^Q zwJef%i9QT`mq8-mKXO{+8OC zgM#Qnahf{i$kX!q1^mxNyle*OwuT%jRQ0u+Qvyu-yl9oS`AV52@9K=N;$t1jm-z7h zr`5b043JHf{PTF(nWrMBFBi87<>b>oRYpe2Z!xxn6^Fb9OB25%%;75^bC$5O2Qxc}9uTlFcsAi)JZc~!VlT0QrxC|&egl{RY9h z>E&g%MZ^(L@b#<}_3{jhHj_9DI%I=0@uQFY){V-{KLi4urK$%-G^hbdr~d0XC7y2) zojtrkP;&VajMBuoy(K#Ml#Y|KBqnU(RcI0X`c6L7Q?3t+7lZzpu&}X`eW^HPV(QvY z2q!0HQIwwDybXEN$|5$xDtTNWE-idlS!k9Q8P{|Am#Cjxz;K1u*z)pNHxCU5BHQ=R zq}$`uO7}lr%#blaFsGpcCEa>{OJ|BhW?D?s_m58A3H;a_yo8Qx*d{2UdGRzjtcwU{ z6aIApcT`@S{F8TV}>^OJr=w$^`FVs?>drrk;heUpZKC# zc5X81Q*W@ea&|u;)U+oqe>@JO6hps!(OkMG{6bPI;Dw#?BW`##|H@X4qua7P$^!%I zKK2Yue*)!qZhsv(gBSnH-ikvwtrw^D?euqWbUt?ZXG%9~Xuq}&3JPSUm%jgzmJZvO01 zY^MX<8+ZZuef`Tg%Xbp{*aMpb&l}VX`4zKMv@Ip|B1qp@l2qhi{o^8LfStZ#Fnd!1 zpa|EV@eDK0CmZ9ODnR1=m6HEMki`p)4}hwJ8f<8$xDNj;NzeB2@i`^KqFGCWm$9J0uk{MReGoa`?oaO0oyA=!+;QdKOs1%Hy*Bxe}w$#lN@o)0mN^vLy7iolY?@ zhV2|2diLp^l1E#_HCXb%k4q-GMvQ-WezDl-kAsp8TW5UHcJ#U$xDfZpG34JLxCcEu zN!Q*n7&G(vP0sDGutCnut<6uLN;k8DX3JQc$2^Ma;DqY>05XPK!XtUAR`;e*$YgI- zg+TOd)bboLOCheo_5Uj4d(zDpi>>wh$l)IcV@hVEn~PpXO*=E_tYwx+0W0E3WCNDj zZdXT@XS4)!aFL@}tLqdGWwInEg<`s+W}*;XCYbKbK7X1l9~Any_N9_)xu6%KH@s!Y zdz3LSSQo(#JZw9`2@5yuxZjfIyBwo3^~RA^%7ud)BS2A!DL#<6zSTImS7z%cpt0a=>WG`D(m&f znt{o%y`a2D4jH9#B43<4rmC=As?=25xLi@Ts)mn?=!hQ}Q1A2GtHQW)O8g@{&EtsB`4Q zT-jB>%8X>?^qy+?`;HwgHQW-P-t<`I`CHH%y@NimN)IYdq#8O+MYMU;V;edAp? zvrNdjjJ(}bsz$FO@>0T^4x+@qU<^MixaC`_wqA1Sy%KBvQ00isU>HW&@=@-ck`p96 z5*YcK(b50M@N4c>+?M!O#Io6WtDt1W>=}!Wj0dp_H0O0Y^r|BQ#LhW zENiAtdTP3kK9YZmCGqlES7eh2p<;3JK0>2S7mGvgN;mU+@fv~Quc<=e{}>uH^43Db z1Vgv@=z>>RINB{Nb6(RF11|h>Bg?njmAwv$)~O?l>>{myPxSkeki<}`S>N>Pd@*iN zfRjB0P8PvL#wrv%Uf$LFze|581)ycc*aoTx>zlX$Q@;-c_On8x zxKT3id}Hi@^N@r>-sdZ!&mlySU1@2x+NZHs_)?L4=sa!xG_oOcyW;ZgGPGmQWbA`) z+oy?3p6vH}eUVpMl6JHNNoLmh5z@)zHuA(1OJgB707hQjv6|HHqo5xC{~c%SGMl2Y z;%w(idrL$t_xD|ldg{!-!FEDR&%684s}EPs9H-8B9(R2e^+BdWhVNd<1oVXVg61>4 z&UMEn^7>Puwcu=zq@`^T(fOH?V~~|6-9)^Pt?iE_t~jz@GHb~1l2oq3Y-~5X^=@jl z7TH!9)g>u;?hjZaPF;r&9tHLeK4xN_yKE*FaJoACR-s58w@UbhiO{GqSZ3}I%Utza7GP@a z-Vy=afVNgd2e|GJqsn+%LBB7GhK0gm#oisQso^G|*P!Ccy2;tQnm))Htsd-F%;On4BuqGg|x*58e%@vMajP=ZH}m==sBqt4OB+< z5c9?utY0^S@U~yn;+8>5!|?R~Y*5?4EN(8hIN#-a`$URY^w||`qacdL0H+*5AF97_ z4K_@zSy3x@?5A*FcC0g^V*T4|!*8dMy`9&ad|Hx-GPSjC_6PylyhywLagozoDsPPZ z7N(w0Ecq5lcRP)WteEH#Y*nnX0OzdPG$Z1@<^C6?ph2C`xuJPd{Skc-rrw>jdbZCX zWqG+jro`$W9MnxPM*kc&@f_5~S1$x{bNJnm)B%iBoTAgU6P@%c-_*D^&Sa(YY3_@@ znfIys=oaB4TU0`n!S|fg^A?aF!W^eGUWsGp`M2K~wR)2agDARIaXZ()H{z{?1k;~Y z1qbCrYmdm|BfZ*R_lh4lY%}TL-vkfmCF`C(xfUCv4N_(_m|QMEcQ!_Bb!>wgASuDm zQ(`r;V!%50%%Q>UN^2cD0CTRYCBVUijtRQGWMUUA!MhwI_GrU@Ye(t;AZbUwBzhvsf zwwk_JdDUtuy@$@1c;TAcw=GFOQJIIj(!qb=0s_wIhdJNodju<6+$(E^EzR+>-E(3Q z9^Jf!WeRV3K4QWZ7MiZ>^V2GdTEWAFP^r~q0^mkP2BiWxs@1JM4qQWKU}rZ!Lcv9I zYejgFV_z`=Ikxa^!HmP~CG%Us*?@%bwA1%Wdstpxsy={2K$1wjv#dM6Iq z22LKOI8=8yRceL6-JYYKzqjHSy#c?Zd_Tp||2CbWHZu3+&653b-l8|_VSyIwA3@Hbze+#nH_H_$`A{x^?F2^zSYNhd%>TOmc2Mx z75Nii;&gCk>#G`{+Yj(z<%^cK=t^c^{n&F`*kRSlhU59-I*^GtPrTR#7E zy&>lVE$rPzUhLh`Ilh`cct7+a>1}mez8=NBox9-CYRH>GB=9Jx`)4>yDFu%(hY+1_ zjs5ZL+uFgBS~42g490+7+#4i--amqf&(bU3oP#_TnVMIH{NO^m{C$86@k@WVZMg$q zfnwnndr=-U{n52&X)0=6MK$fGA5<%}eq6NDuN=gwl6?g>!I2k667F4ibLrJY?B@FG zryM_rG&onc;dQS3rZfB18;~(Kzc96TjNDXYNgi&vIQ2S&sI&N$K327v-AoN#9=Knk zGrm8)WY959`%JU5d@nTe(}Z@2Q>>U7Y`Q}8qm+;89cAVx_EY=i5d7lU!`i&-@5l8* z_>xB>#cM1M+cFNr94iJzEu%;g-&A?%#Rd6}gEW{<*Xal4;~z&~i|KN5$UPmJS3Zx| z`;}W%ZgX47V6Z7Hgi{x1~JG=Kdvs-&bU#ep`PH#X)#Jm3*I=#bx z@||g0N~J*kiRRE=P6e}W(!(X4A)yrjEaGoI;)>A9C-NX4_!rn>kY?Ol#g#-7_)Gwh zD@*g|Z`$`QK7KIWMz^S5Z&Jz56+Uw-_({s>ZYz+#yD5uoON$lAdv@h<#Rq&wj*?mm zMky$R`I77*8NPVu%WJ-5aMxeg+5Mi%V%BMCxgqUM6p- zFVDNQNPaoY7HrN^0i2wjhKm>YmLhzMF#Km|@ML6He@Ugfxa;}$9a1s?$LRTYmU^2S zq+k@pETg>M${ubI#h5>;{tyv9Z8iJuaP?d1=2bG_FK*sRe}xW#;MQ}we1Et=EB(-g zsV{|sx=Sw}EHk7w^#>9MuK7K_YL#|_lJATkY<1cb9an9_mt6LnWh{Ta;bnk6s2W07GN(~S?j^G^rw&AaK9ySSiD2esygyR$$&3)rhxlR@?lX>*2VsHxd$Tgm}j?3Tv6it8ml<)5!5%Ri_Gn4;C77kd~; z)zMBTlqIj{9jBnWIe~KhIAI#*J9(g}#zC1S2C|p`g8~CCVliFNx0c)3?LH&!cwk5E zPi|AMzj=Oba9`R?H*H928J$FTUaa>+J`vto<)(O@V?&dior*Iw;fo)Vkf%x&ONeUx zj5k%&L%nNtaU%#2l%ZDPAH*lZ`g-Cs!rBIZI^?pLkUWV^Kbelc&6h!DNgjnpd_ERh z^V!2o%A{o|>JvOAJ^22E0@lMK6(@w=+@cy2If&WJ)YlAUfgN4xDd#ryN7)7==?T&X zDI!E}ymxWiTH@=lK`e1aY>>jMc+|Q-qVlX`oX23*L|q}ncDd!)8w~6iIM0i}MZ16H z$bnr2XY6PxOO4$QKc0gMK4t4PuG%uNM{MwQuMfp>UsB3&!yCT%JVW1VH^rwZbGMg3 zWv3i}q3(5%g~r&iNEe*L98#SKsaJNw z2Y*f+92(eL53kZb)yy(!IfPj(n7_Z-VxG@6^T6_(?b!2H{72ndiGE&NeC?x{f<2Vk z)}iy_MVk2$Z(mGKw+`<|hQ= zc6r>Q@=^8O;1{^UTTJS?|;o| z;=aNQUs~cz;YujTal;g=QV3UXyH-W=&mV78WhRIFgj9%Q$*y%}SF#pA(g1ge*(f!t zvPisX+BVXcM|fOK{TgNd=(Q&qhj8I-WUKA|X|>$)kMVE&7Jr({?BH{MvHcl`2-VbF z71k{CkplhgXs9W0v8++D*}@%f+tV6nK0H%3T{lv$9h^aM32h617gTw*qzEx3p)e2@ z8rcT`rxneyxhJ@=Cei1-fOLF zS!kY-HIPJUG#xH9n$O5m$h+g0RH>t!E$(#xGMed4xnI2y@xLwp?3nV>TnkP4)414D z6Ep96v)?+mFU23<%&a(wwOCz$vEgOIEx-9jB^$<}VBH=>Sl9;29;?{x%Wk%3`fPQ4 zSsUQYeyby0z(1Cqx039WfE}wOxBk+@n;i&mdrgL^-(G6rONmN!$33=^+{JyTWY}7+ zHJ?J$@su(N?z=mgT%$+FMvi$4Fx7x3-&3JvBQ|V)rmvBNx&X?~gKpH=l z0O^WauqzI22qb^NUxGqcb`Z0EY;TPED7PpoK;OR7kHvQe49rOFF{%FXLece7%u|=U zV}?rZ&D-JlF-~^H?ZYy$3;ahXyhnR#uFnvbva7~MN8TmZ-Gg|7ZzG1RO_^BH_p zec*__z<>Xx>r#dT6)(=#Xhk+EXd_mV&zsSR|E@A2W9`&X`}v{fr16Q7FiPD*mj73Wd0Gxu(`lx6XQn}D5T4qn0>$R_I($Hn`#kDr!$CnC%BcF=} z95_+-7B;Ambz5PtS!e*RwDYc`t2EK39%lnz(k`PsI6?+kakgjr9YxmOxft^4V(#6^ z?TPuPmL5Jx<=VG9A7>jAT6qY-(3>s8=wpRE@( zz9o*|M--d3h577o%)3}>b6R~NGS$w!^gTS^8!7rH2h{zBnJ%=Ieo6KPMJ* z-LL+#dlwcwZD(-TzVcRt00-b(Tjgj$fv{#ncoTx%$>|nV==MN1)ncf>fH~qj%!=wg zW)O?TRWl*8>p&~#qv=FU%uEZ4O2M< zvq`yeLl8x*mmk(`^`?TqqfkY5H z2<+Y!FZ$wu!voptPe8Ix?CYO4pFgV@p`AgbmfNGj_Nfp^L&Nl(ke+Br=VUYefU8%wc9 z^YHlHCUTX;ka_rnS;N?@!|mM(@`jZT=`Ou{GTOUc8x^)ig5wGQIv3;WT@&{%j}Gsc zl*I-lWB6j=M{uRuw_ya7ANG@1_i67N7P(*c2v<7>RJ7b>+ReSNPQ`-EvtGujl`2Gpm@tdH(xD;XXXg z*KbDLr=PYwzA7zb9KZEPp(KjxYXSPSTIL(?K+5)LpRhJB_2oR0Nw1(7q4f@%YCr=f zms=_>^Dq{+O=b&~Rw{i*L^EBd^8@T@6Oi|6=*JHjIMhMk ze-S$U5H|vRpidscb=!)Tj0?s-J|0JW_PMnYcb{MMCBRpce}ZqX0K#3Nnu++;GPTOB z2&DB)GEiM5XXI+%{l~}T(-5$#JBpf-iwRv>k4jB9ivH{4O!xas=5|5^#QF1s{3^d3 zU*HcFvoV&!`RHxvE#VKnZ8vJDJW&nu^Dvp3o6Vn?#RuW<4@hR!HOTNKO-vLc-nT&G zwF^J_Tp!dz1z#8Th-?fhdS@2eUWydDjO^-hgj9+=Q3o#V#tz42HF^}9z7gP#5Atu0 z%opkF^|sw9g5x2%Xa8-W_1+UFK?|5Wz4EVQWNTW3!ffi5Bf{FLquR^xpNVKRXK^%0wRKMoNCx=_5{*I& zKY=~w1*95A)niklR4o`6HS0YBB=N$(NaDE_?-8GX-vLnDGm%#+q6=;kG<*EB`7|~F zj%j|-=#!VwBd;BMPp@d%ctF51UZEX#pRmRL7pX@J^n?4bQA1~Xlguiomjo0YYQ3t4 z>JgT{V(p-oHi0L3|Jj^&ilFC*G4|A><$my{};haP@D{F z&~_a8{M2+vHTOE$6=`nFL|{b66X5~$<6~R4isJa57FHHtxo%-4CnU3(wnghKQSkKk zA@5RUyFkvTAK%^8xY+VqUY^Pqbwk)v$HfwBKC;CXO2}zsf0FP1IYNe}(&QHm@Pi?c zsz^o6$NzRdwroP-OX#6jyG<09)Y`UGW7!MUaTDe=mUQ0<%6RBx2`f%5E^(VtE&zPkGjmev3gV~=3&}bZ6jataT zJ}SN5#jOlZ4#?vbZzz#3rFByTMVdZYcqA!0H$n*&KkWNAGXp}wIic(h>ktRRs>F>K zX}d4Q^B=Q4GjUkSZzkh_=%33bAu~EylP_PlT3i(nYX7gI@w&h`* znJ399dv)O|d=OTYVBi_pcBy)$`QUun0z0ds_LiJ=^@mGbii#3566RD5{Z)z)E*A_x zM!zqeF4L)At$Cug7#S$$Tu5-2tM-T8mIRyzfTD(ukEJO10G;I$M+X|mq z5>*%z(sUvSYI(@jav_XzF)cAc8+`>wN@1KwWZU_qdARZ2%u?-Y01wV62+3NwYfpf> zXt(_3O5ab*Z6z`G#}YL?d-8Ba7Rz-K2Jx#;5d6>X{!565UUWy-JGaN-5drPB5;>ni zz6R2}4K$clyjM{s!DeQUkbYx^xy@c#l~_mfgZpgbV_T$U4d%x1$kkdYncMv#b2S3! z`INAGnlAObSLJv{M;@U^(A+ByV50bRx;;5VOxrzs*d%D7@(FOyOP}stl!{$Izl4jF z8ezY{)7RRy0LGvLKot05tIh%KcI(6I(MO1NztWZ4Ubhrx1|54ZnrD zI$_`u)R?@aRn?a%`)svPHo-UG#WwVWTx=tK)MvIDYAmHQu|2L4oR1!bkAA!_+NNdv zD=5d3m2VG{wghTb8)Yd3Jhu#>@vqZc#8yBgmvRh5axUQSd#lIP97i(j%_eOtGV3@j zq^E$E;=hG*7+*?nh0z+h!R7>jWVxn|X-mr#b^eZ6b^kzMh+u72O7@9E!`@NeQM23m zw`P3=Ia2Js_5)dtNW^T7w6@G1nF|2yYr5BbeFaBCzPch#5i%zH2&w8DPuDQstYZ+6 z?n}#h`dWUEWxIWS&g#Mk(?br$onD5xRSm=V$z!-I^|%3n>2n|L26L`k0*_topRzJO zxs^N}g%+EXJs)f4gpN#-KFjGsrA>22m^vJ#;Ex+Q-W`JX_t@)+q3n*8xgx{V0FD69 z4%A+`sJ;wRiTi25x~lXQ)(|^Tf!fm^6vD>C&+Z?T@lZtnXQzN zKvp=f-Dy*f2d;_$f>2YyYRU#Ygk~?rL6~xIu!aJEdCk3=inQ`1u266G(Mnt!% zeC+%3!S(;tmUCW-t#L6cqy~@nq4QxmzeO>ze6gli)SYht^n>$|d1FVG^fMWu)A4p7 z+S?xL;E$;b-nRTEs79s;8id1kZ#+q=};r>r*p_A2V31Ona9{|}S znEC!i-&(UMc&DECN%#Lt-u5I`Jp#F!Z$_4{3Ic=X031pI*f=?v>}_I{12Wx}H|kuW z*kM88pyo&10}{0L@VH3exOS{#5yHZlB5PCwuSu*N|1aZwuwL7XfaFWZ(iaBO*mYr@ zsvg7pBPpVdwuICadkYMOoI8F0wOB%#wa)D{kfHGE6(Lg$7?2d?sahUj7#OL(_BnC> z*g}lCY-QD2iDW?Xl#XvZV9gR+h3+~eI8BKSru~WPq=n`CYF4KBDls`Ue zxdb*kX7@vu`H6iaFrV4^^usu>;dARI_!8iLlsPPa^w0?N8kA4ihIfX>S0WPp@b$%y z0E=R&6}V?Hb_3Lf$|3Y4gJycA3ppcG*@L z%K)*8DXM+zQpxRme@sK@FVlEQNzO7LahXJq;H4pYQK%<7g-GtOsbVMW{*~4Iz|{B4 z)`LxsmSQq`VlQZ0j}FfRe3{@_Vb{{I+%nm@%op{54>dYgb0i>Hr1GbU00X{QfV@4c zn%yjwJ|x=Fwg(p95~ZzTBl~#)UsL0>T}EIyNBSCF$qZS_Oy#dHn?iA&Ml%ZgarpjEr|kVD^lkVhNaS9u-E;KJi-B>h^?}o?5T;Y&VWdXZ;KG9t5JGb z#7E8Giz}?#E%}hTB|Z#`b~Gw&mfCW}#G(<8Y3G&q{=NL|Aq8WbTv_zi z-LS?CTd3X0o{>YPtM6#Io59z2M8j(rf76mF2J3}%2? zAqb?Mj6u#%rNeY1)FiSTtxu>IYhn zz=+rGs>zuI=My$$IamJs^#tbxggli2jqGJtp+!AMz)6P49eRB=cr&Pv9qa)h*37_{ zJ;sF}_h=suzHT`X9M!QsVi!8F`EjrIG*m8qf&aMUs+10**#_(D{>=RCTZmM#$gw*j zw|@(8wFI}4stcOUFV1f3A$fKIG8W(h zJlFVUD!6|!6<4Hdbb3bj;7E z-)r4-CaoAz^~oeY%c$2UFKXDSa$4wOHNpqCxIOkh*}|&+b0Na1s=0SonkZ1HDB^x80}mmzwFVhquIM)eY_GlS31N`OyBt6Ns$? z2EndJ0zTW6u&)%?HG4xOzaUdnO09fMUqA2gJkM$3;`Foh*E4g*= zQJ^HDUgTH9Q3eV(@=ll7%&O&XQhZbs{Bl(;peT02hy5bp4!}P|Mh+?3$aC(0Tm*Ci zEay#Wosv^vo$RC7lD7)eDq9GWLdGq$m}$ns1Tm_k5>9k94@e8G&=P-jM6bgV5Ez3& zc^4KG#CzI=_c@*Nmr&#@pZFn!ZOZ79j;EdSY?-GOb=g$qEZnBX6@KcG*7l8R*dE`6 zzen0ax@)>son(oeL_nt0aFQW;8_!%+ z5AFs0`24-V{v z(&HsO0=Ta?A0XyB{M(-ECC>V23Vf;E!vc;K|8z?TvE=X z^ZeSowFJ5Ur_;Eeb(C}O4=o85U6xg2cxO41AGaf}=32K+JtGnL&gq15AADD$bO1fM z>L$Jm^zxgIGapXr?F8*7T(8sn`Nv;;s*q}SAGIyEo6g7s-~u(zD_e`ylL=d>Hfvtn zJcTDFXf^jZgZpJX-Jn%eq8a^T5ZS!Yk|S3`Ox&=I(m=KsME;cREL&Z0zw`P?&srT? zG_v31qP_tl9Qi`6pxs`I%MakTypAKTtv+_tSettsSJRb*z>mP4<%r%h(|c;VQugVF zcKsYbak4@aetu|vq}OLQ_@ksvPDKFK(XzLG`U7KiqF2QhSuQR>x>jbp;SM)`9Vn;c23~){ z?TI>PhRZcG2WPc3cdROdQww5LiaLkt5AMwS{u8$$7H@LZy*`yg0Xw^@5YTUi9`x>x zXhknFnVft7vkSm*6;lKAE(C`^06PLOrhngKqyrmqHUAp*f9=P3-$ruz1aM9}gTb6_ z`}*M1%P-F5TrRntLI2daD-D~!VQ;^UEH`;0x4zH}PWpi7@ETR2U+6lHks=D&MfCBX zz--p?|30pXuT%ew^^oPRHs06sJSJ$AjH)pPrc`GHAK-&hARoZ4Gg!WL1J37c96fk* zM;=?THZ6g+VR{mX=+)%-A}E4cbKl8Rw!F{~nzr%)!5&E!uQ^ZLXrVlIl3KHvjDd@4#Gte`Xh3r->-YIrA@zWaRZ%*qD5 z|Dlo7X}h2b3K~JT2&;gu5rh}Hx_j<}Lvwv3B!Nfh5m95^M?%YA3ALN${HQ+r@a^f72`VkV6MWU0C z(@`u|&SK1PcUIF7V7 z_x?r98aQY*KXiW!sq%Hj(#PqWbH6izQC9`2ll&qx?y;nRn92q4K-Z@?Nz#k7;4#i_ zCUP2N&c~80 zLy`hP$Js@g<8t2O#p~+yj6keMrBF5e%!_9}bGc)5rIytv!m5jXyA`Cy)Pw-(_jPAOEfX2lg_3LwP9WKFm-iefZO z>%j_VtQg=6*ek;|o!!k3Y78Sq{zW(=YNkM@ngmF(cBcYoG7gxLWwliTyY9t+A5QI9 z6?K}Q=984{w!G~Fkc3gH*`$o7reoK=2~y?FMr8T9^dDTlI~nnHS>7jaQ0{M*>^ zlI2f4@XP7%NX>s+^1H@7TQXi4iwG5D`$74K|GgSp z-v0-K90Q#5ubplD;bOQ!@IJlk{r{G?O_%>=u>U`I>N|i}^?0|FC;~?OUB&p@3La&Y z9K>(7+~HvGZ1_y3%adoB?#xSyQ4xPeZGbZe_4X(ZwAkJZ0i*hw-V92SRUOS z6H=(XUtH+Bn)|n}A8h41KO^4PjUfG;7cdTAY`CYa{2gTb?Q8Gy=zQUV)jJf2qPuOI zz6V3UjWT&<>C83oor)cbrqSJLA=`x`zk{*AwKM)M?LUDk7csd{pG8N8Y@c9!H2$YS zNsf0RcN?QFJPr{Ro{Eh9uJ-e-p=Y}i#YQ=e0?7W3=S#NmR@qR z**LPZK*C7@nCe>T;y!q@Y*GgwKR&S`rJ*+@gOfv+muqNF4y0Umnr_xnq_bI3O=(CK z$_4lUCX3uxMpuTiy4>J{4MWcL6`vL0dUzapL~=192TC)m=$nr24hu0HXBKGG&pvo2&A{`dz2xvMZIv-e z^1TPt&VZTQ{X>jpIX_~-?iG=?NeDYxD!SYG+&{_C=B0*vx)B4LCv>=E&qQl_t+nh! zFqW$F?hwzFm*g^deBzQ(b=wn8OK-7P>6H_Cdkn4#O1%*Yc0J)V%NSFlpFndyg~Hcj zg8Qp5rsHslGihTfWf8Y3L;U?N84t`4qLt}eB<5t|e z$;)h!py+OD?my|k<^iB_xq@Um_eQPNmjRhT5B4E=<6A$c>8y%hj=PD#+hob|M@eql zYn^etHRmhhysM>uPTE|a7kqDygrvxlA4cYUsKSC3&OwO6#u=wx{nPTELC9RhT!bB5 z(pjjr8Xu6M43##mQZkdCB6A>BN5+MU>|pVpK_4{dZ$?pqg3_v-29J<`LX-B7r1%(* z+NUTTO~9!$(20^(t0g1zRejn~V8u1u{;?-pVR5T;LGpbuZm!j*flHPl_?S^!$t{of z%GZO7nmUM0Z`)jmp2PKyA0v)Q0P(!(QMdHq5Ye;z-J}uQhVX9EzyF_9dikMrJNt)j`}gHW^QVs3tuK(0YMI zfPH2l$#TLk4!58De`qYD+4ZNuQ1+qn%7Vn5vKb;8U~-M+?oBm@7lw<9sYihtpEvWHRm%o0LZMte-8YdB~BA z-TC+!u16bfcj4(j7-Z#Kmm7rBm<+5F?n8x+BIX2GswNG?P`@56ryg6Q=lLF_o7yMY zhK%|h$I@esbuA z?g}`Ce?9c@O(R{5t-P)F7UUKT&dbsaV_$-6#Ma0C?f-B5nHzpBS@43in>#jy8mT9h zaqE*ynt}XZztgWPZOH#}4{s#(h=_ekl=#`X;rsJUw+x~4*T6+<|JtmK#Y)1;n~92q zhJ+CJ1OXd=gW`~~6$Z{jagAM?)R~ALae>FI^g&dg1B)ruu@618S2r|XN#*us>Bp~? zcbmk&M99_sbv(wq3e;quTaenaNJR=dt zX^MAl5>c!eVSg^(Z88(Y)P~Pah$V95htk;p{ss@dV3z>Uz9`rmW?AKZXXJf%~{Z{2QIt+}SYnHk**N36Xh)-Non%vG9RL6BV zJCj_scR5+UHCn0x*MLc-ZH0q)qf^p(*n=B1VAN@826)nJ*3)y34@-meH?(b!$&A{q z?K;Q@3i62G6C`zZ#|R|F7;)pI>b;%@`n3)mY8CJ1N%LPux9KJ2b%g43fYS?`U{1ge zTA5hCB#%xCJRel`RCG3CLo9w%Bcqb;4>`2hgv10we_c_xiJmjatGi>Whtx)bdb9%#9MRq+ zj*`zI2z9xPM;#tNr^Wr0t6ZuD8PS%6Ag6KO{a5;OuDvX|%o}af#$_O?8DB_}Kk?R2 zfhsz9Lwl+7>gY=h$mpamb(jGq4od|{R%+T0*CAa}A!U>z^ajd+cmp5;>(n__O!D~Rqk5Pi&kC1vr`_;k z@3XPEimyVJYdAoCT)w%}pwRs)BNV)|~#!^vr{q9`utKI|0%5VN&>jWTyH20$1N2WJZb#Nj<{@H=9dTY+;FsVUg&b`MLv3&DAg8_VT%>hs#)jIapW1Db zVdafilCX5)h_3^6=}vLn7ftB*ElM^>uYRSct5x78T+r8>+zpKD=`q$)8_$MbXd~$P zKQGJX0ADtKw0xt-k*fs!fw0c#$sBVVT`7g`6i2&Yf725eLg^RszVC9Qs~fIR<-U6@ zC{V5pUo{ziw1JMF*s%S9M;>z#ioXIAxnurqSCbPVO69r2<=aB~;ox{a`@ouEY~Pc> z7mp9OOX@fnYH>}<#y;qUi8R&f0;>yfu+x61LUZM*@`K6`t%_WEW~J%u)*_629x7A5 zWc7!Bi-%>86s1>WBse`XuGwosA6<4UN4BW&5VUPd>q)`LqYPx)HeFzSf74tZ3T3>I zssPb<$V;o=5|+T_L4o+!QZ!GnyydUJ@B??JBqdj^^_Jy+@gPYFl>5lM38NNX2t3-7kJoRVE|Z6tQCh?wqKa zn(AUV(xWu4Mm|KC?B!i>^a@<3>8evsh0i~Koj>U(Tu~5B81d>|Y;??QQ!oe;8!kCu z=&_x!7#2(f-!2|Euw39zj9{C_FPz7wLWeN&kD53W$`k|2#Sn#pN%T$1t^=(S`~_(S zC*4wvL|y195w#VT5bLExM(ca+Elfi8-p+2MZ%t{P{v^*XDF5rS-LFl=e5IFMG=)>7*gZzY~6$XE#L442e##CirhyjSS>W{9sy zQi7Vj?4Zxw|4UG@?Zw)83`Aup2jv65}oS_MVASoDJd^9&}Q;26Fb>OvE@ zoRoc~UKkfzSgs&m{~Dg_Kmy}3=36PvLF|aBl>Z2UP0c;HjD7j^z|&jOFhk497{4UB zr>ras_cRFt*M(;DDte&`g6i+CTZ6o!K`%F0O&%`|-9B`i`7s;Q$7(%~O}sj^Iy9*U zBU-Ng!UJb7pI-;BfyM{0@4B3ay-exso>4_2nh!XE3HSf_yVIj-;t)|KjLq)t3S|)B zq!Z+osmI48$W>7Zb@D}fePkHF(%=-{{ZAht)pjOreh=I-ZLJ|0<(&1*;TsDi=s` z`c{I_386IXNS5G-5J;SSoHwU5B=GfOgtr9rQaN|0Nw<8n6;UQ{TwpuoBq|b)$_%yS z9?QcbPjcK6-OJPNk7{Z|+E^=5!XzesU?rLh2KlZ00c^lQTaocT z0I>V(d&LoFB0_&KV4bz3q02EkS5P}u(=hU_<)I3O!lWLS{uBr7Gg=}p^6D=z@C5dx z1O*;FA%JpYoA(tFU?1kyFX|KK_JIcphZLggEwO1>o=~4Q+sJ$VobF`xNu!fXFKgwi z+CpkMl1{h?EilbA3n@n_lkyh~BGyBA_Z-(8l_vB7X)uGirwluv8Pm@;C>c0vaD3(dIn%sM`;^5k=Oxk%E|+uiDdse+ z8%N}K3D=}jL#ml;ul5?GRonnHcqcw{{rrot$wCc>XV81PUMHz0mY_#o2_95A=RhJ4 zRkzg^k9k!#)YuKW80xGXvcKoThqAnx$|_c0?4Nung46uY*lu}O9sSI|t{L60LL=x| zL6OLbpi2^rUKfN-2E4#A*vcB>EA0JO;Z>$%s1CiVK`FAd=jH_OUY72z;lf*Hc725E zC;bf|(7hA?0zvWV0or*yZUP6*g6@^@hFtMDgb)e`MkMmmL>QSXM{Sk*2<*HiHgNKHj*BC_qu8IX7RkFqi1An0 z4@`D0Ntly!%F$QK>MCh!R$@9$b6IM6P(iDPAW7GT6gQG|4uSh&AwS!wmmRxMg_^@? zhfb**?NG&M1|Ee;%ko4U_lIb6HZ*yXzUJz<{La1LXNB&4@U~7e(y0E!kpEi5Pi)qF zB!~PZCQX%0nG&0z`OJq$Z9f$m82&9n>-W(*Xn-m_zuXJnuqr@vE}}4LuAd9b#9?Oc z@(>MCS9V?xP+ZaAf+Fz(sHSnwRi#)N{$;NKJUQQV6=AkfD#E;QSRRD+V&eXNV6?^z$D6)R-u1-VRy32e=c zb+u2=T^ho+d6Y?m93Repg4S*6V?jP=cA8`$L_d>x$L=uy`Q~X!>@C1iJ@x070*DMD zq-wHbqFMEM3CY4u`xN+>R%)6sricx#mI~P&AR9IBug?74@_tL#pH2$Lykmu$s1A?u zeJUemiQS>QKiLRVn>=`^ zG9~znTSnCln4<<71~|ZAf5zsQ$2rugOuE=#JU(0~4gw5uDjS%$JG&v!kA3wrM-h1i z%%}&ip;uyB-q$O)R{}i|^kye7@LhGM^HvUp>C9X@%3lE0ieYjt2Q^@Hj=fy>*81#z zmT0(sMPk2pguc!igc#!x$b01EPW;P1%-q_BnM<=he*TfwinRvcnSrNDS}Hs&(U0L&cC$}~v5?QXzc@FM zvIWGta!TZ?vi^JTrshY_5a0bNAhe*<1(52a+0j%1f|_^qN)BYdk^%AsGtPx0U3b(> z8fJ}pP3&Xo_p(}PIlQ&dC3>UHZ%+IdF18mG=7sJn11O+Lh8}H zIv(}p&e=a=mbHzTr3AWWk~v$V%~SdtTsJa}oF9QtdXO)ws_J>Wan*Osz{wJib85~u z?`Yprkys--BoogR{$qa8gWg z$4_UfQwN=A_jfh z&Qd2)*>=z?WA&MC(ETy>?+2c%@s>1H%8<}4g9&Yl#@HKEoio=BB_;|4QcU8)z7DkBM%Fj#C>&NkH6;!^IJm<2usnt^p)eQrzV8DjlFpywJdMXt%}}rL(x}Cf0T=v3-U<{Rx%w9s zFxgOnAiIiJ-)lfRqUTirihJVKzt3NyQ)&io`aeYkL;JwuWfGfYH;VKi&sq9vGX(5H zLFESTDPr}jrMHJNG6;bAVbs(E%JoBjrIXDM$jg1eU+_Ec0~y}a!1;4cLRlZZE#}q0 z9io5S|E2sqllnuchqAcsx$vVA`bBO1`ozdg6!l`7B3|X06B7 z?>6|NNt@XAa&bxoPZ``e5(Mm6{rl~cLYH8{z^3-I^ikhzszEtwgkFirV=9?$cu75X z$-MD03yR8^wou%yvYqAi;O5IsrwulE03g=qAlFly$z4jJZX*HSwDYk5>UWc~DDDCU zx7jxziP*a9cEK#^7R`Wbw*xMg?^VcIj}0R2*WKci{oiW1={@4M0?+)8TrNE00uMKc zr;0=Dx3`{*s&qev8cm0R+-5)OPv3~49jyzC3pE7kt2Z0C$!$w>ATd0>m1Z#DC1Ln= zn&=CGvhKv79<1cPF8OBg1O(5}nHFT~q>n91gq8u%&b*H|mEV@N>8wtt2-obEw5b&sf>kdY8 zj8Vv~_ulI%I@;vrw>#Xv?qQ~c?S0<-fHbywP1_gXEo_kLg7Y=HtY+p@k0zu2`^-Rq zTeCTC!On3#^fL!xT2hX@D{@SmA9(lynj1nQu7d}zfgo@C;Sq1B;DZ4>6q6rB{O!YH z+pHc&U}bs6JKoPqY(se^4zMc`kg1r3Dj!tjxgoLe!>%>p(NvmQr%J~0;f74u_OPJo zBcR3iKI;X404Ig6i9Nd3uR=)lqFE_9^~wf$JG6lC-U;5dx(8fr5jI^Jc4>`K_B3$x z2tX}Hj+}oXnY#Q(Xn*19H~z|d7gQlH+Y!f$VFDc>`wOBB>Lh4-@$NDZ{4>mwuKOhF zD1{SMv-yj7rxGv3@u(^HFuDTg7=USOXh(uQFXwFuhw#{WgGiti)P|>qqgRz&m%= z`8>SjWo;vQF+*Z{o+SF_TZtr|wC9zX>KyytBxpdZNXbF-_AQzpY@%lUqdBKo3p(4r zM)RQR@`nvl>cxwFepkK)wq4GZ+E+OI!DqP1Rx&0{F;cw1$X7U&m)9vn@N=azN%0u_ z=l!Z16NS)k5Nys9oR38gTthh~oux{*5yZgww$k|H4G8E@;)L6B-l+P$IUJL?MTwI$ z1~0+)7A5*%Mg+)^n3`(_(HlZceaZ7@=s)&4v(DVRRUr>bm@^<|zmAGo*mA`N$arR_ z{N>(Ynee0>&<69iz6K<|8+o#I2k6cvaGe%WyW@PrngD?CglRGsbCm#(GA&nAbZ@y- zJvih~itz6^M}E(MCoO`y5lG%UM<;WB`G=(d=J+7Z>a3Q8t%f)LOmAVgc&BQC=TLbR zCe#}h>E7#?BGY=vO${pnnUY4aoPmUk!^}ar+FwLig1XQoTw7ZK1W~#O!im$m<Uy`3P+I&3*MI$Tx*oN@cyaolbRNbbMeK+kii6S9$>~oQh zqP!^oZ{{r5*{#a|GZcNXntTX?!WQ*V#awA)g(Z(L2o&d$hdhjwK}*2|UZ{|u^G-N7G8QK8NLV9h}>y5lW&>FS2@@`)5Nd%T; z*t^%8^9sDj0n*CxkGkxmbrqqgMI(3mPK1s_G6vR?M@|5Xv`N-zoq6`1Q~SkeT16~{ zyGscaPd*%O!(41A%aK1REK1rFSuqwDmb9oavXkL3%gBpL7`9g3rXK%)`fVoR5{US- zGs^>_SI)cAY*5%3e^?H9*1R=uvgAM>XFdiS6*mo)pxYDtK`vNC$m{`mY{+H+{%n|p z!x(vUIp#69-WI=brTx%$>PoX3trD05U=nUuazE{j%skK#RBs-N!Y28ye&%JoPg=N7 z`zIR+8+)((_q|UI1+<4z>-^3HNJZXxlxy_icB78k;i0wh5wF^~2V=kx2~R+D&s?dq zHs8na-6dqA&AtJj&TCLDx)tpBuB77}D3ATcemlwINROz-jiVFNk6 z4t3W|qo}96Tf5A0?7gb3GQ;Prg(!O`4G!SiU%n>}4g%zYEF%kVNgVHXuZ-y^5Q|G#at(G|K2+hFeB-C^w!quih z2D(3gFxw!04nWw7L*U_%B$dd4!jZgjaIc;* z+oP>&QZ5SKQvE-?dBKiS5U7KY47q!}DpVCo_4-~0r z0yh4gKMH#_Pk%|0&?Vt2rw9P>o%o{X+WTGepYEwZ5=x$3DZPBi9%m6JLR4m+Ito_f zv_a20f%RM9YW;GY7@&C*xVWpbv6mi$D)anf;^ieJaM#KG%T~Issj-LK%R1`O=Lkw3yO`P8m!YP8qheA`FKPntlZL zP_#0Jv;s8Q)00ENGecESau+qOR*4v##z>w_W^@Sj)xtdqr-HfF!2!>9(r_1?jGN&z;6m9tv>mEfSviG;QG? zs^i)($=nK8JYa2Tehr!=so1sUK%}l07 z8M&+M5LJ^r*vR~wmN5zp=?e`yUvx2W(l4t`u}XL4pnTw2f&&EmaC4BGv%EltppD?oMfft1ReThz((8TyeYXdNlJ zYJ0LowWHb_x&wB_5y%Q_bzF;}Zs+rH}}R&-u39npTk zD>yqHVQJK%!TuCks&jl<5%OBec1t}qz0%sVOOy$EgVOM?<9ozbL#kI6gL_p~3*(Qa zeI%q6F$}L3iXbir-C$1kF|OKK)JbSH_!tD;pmly|ASI_lYs!P_SXJ}!3r)!%Dz2=h ztRbwKBl6c)9xT|%d(tMt@;&qC4Jcvmo<`jTTpuQ(DAOR{(+E86g1bLXK(qeO7XHW;bHW8Nd52O{7b%FXfelCtta=S^4&I{gr5}tnXWZGqz z5^J5A3w@Ls&*V#K)srt4+K{YktZ|{jPRaV&>nt47_Of)2++vH4_sul@*w*;I=BLEI zfv%W3#W9brt82sYBOlkg!&ezYX0*XJowP!;wcyDqKN+_S#NfLQ-3M)dtmXUSBJ#|B zJ|2f4q2LRBz3YsLx8V(g6+0*Tl90cqo?A?HfvPAfncPtPL!#pv?gb|+W);nXQxbOu zBe+gF+X(gH!qkk867Dm+Xi+oG!7k>2DokgVey#Hf;(!&3#O_$rZWs zs)Ih^-5vw}>4Z*3!)S20J}e@hzjY}`H}IqZrM<^IpdpT{;!xal;vv7J)bSk0th*r@2$in!X zM(@Ojq~Hhb?kzC690Mx-zxd#f8@BKGr3-msyt3i?=R{8XO@apyJ+S`+li8*j(|eAY zk0khG8v;95x;=Da<%;U^mbfz(679pf?9%AO@U1bxw0!&ZO&xiq3D~60vhLrSvFF=Y zq+*Zt>BrgQDB%SSZkA1?EFpZn=~NWyokV*;n7x46JU8MW_a*9z6wbA7XsuxEMinet zv0&#!gkDr;8Buz4f7dOs%V={_G21m(Kh`{E35t=Yjw3X_6HVS=?x-_q#~sfUplj%C zHwzEz%uD&YffHZcdnrniO($=qX{Fj(R5=D5FmLFx4p*c%xIhQm*#1r#e+nC~75LQ1 zdLvPxt7g1`%zw#4CUxNhi)3=>$t?ZKhwvP(w5?^8RMv-}kRsicRIlk9`Xk7m`qwc_ zwC`1n*BH4!CqI_2+|I-1)XsRKpj1w`M-EQ9h$MT$@c4>!b(-y(7l9Bs|D55O5`0{% zW-fY!jenxGIO^VVpkoXra>j}oN(R&?b$N(g7E)b1kyV~G!mhDbg~rJx*D6_1MBRqIGfN__wxg`%E9#gjH`-v)KI9H3U@b38!7eSk*TExYvxEjbE~5uXDXGvZ=9GDO zk#fuHtMsCJ1sFYfD?Z2qwPcGCqmFYL3qvf8-nf@V`jp#pSLC3wXAA9Q zuU(jE=Thqw+^^0nBz-8!$5B>35PQp##2`_g!--wVgFkm8K$j*XMm^g6sHI68iy#t5 z)gQ7E&8%tMaD&f9+jsSW?*<_L=9I>P%COrk2;P7AopVu!g)%<2Ae_ zm6sH%HVdfz{7-&h@hRx#VP zVffK^mDRd^;C9`R5q4rLA=mjSv<+PiKR(m!QR$AbnG!tH?C%eEc&5?!m&@2US`%6r znwO8+-QX`TN{iMna4g}`(}hbWKG3o&Lz$&&{W{+=0)wWkj2j|N}3&UGiBG7ed zrDd*nygxp4{pdcG>#DFpX%i*1sSbKFWouv8P-cl*zXG|8PV%16$?tJ@p(nGq_61B4 zL>fDl&L3d~oAYbbud-q&FWa^~3kvl>DCBmp*;D>x=Fa`Sfx4p+T8A==)yL$1tMUex z-W{q%{OETpeV+dQqjOjP_{t3mcKxy0HUQ@Sv6nKa-~J~X{9j3ixF?)q_t9rMqe|qd zi^LgR?S;8Rt=h_-awV`e+$nW*s;Pk28GQ#C;k4!&n|-VPl8fSHKv^EjecIGUbQ4Cm zG`3mUXhiq>|0d4Ey$3Cl+(Q-U??evzU1V%l>XVqfbbK+4@qTLC7=Xdx#l*aBe0cN zdY!=4Z)@soYfp;;y~oNLb>$L`6jgwgEg5vlPoMPuL|T1d65xt&9xjCR!8FdDeG@Ob zO>y4ZgI790NnQ+Z2mrbg1g^_{_dh=NQuY42{NEag*uH~3tqMJU{e2`sg{-p#r@iW# z=`2PO2fumK(P}Zx=d2a6ogS^t7ySbZ@sEYtK=<9A(=a|*%Ejz!ku;35%D9~bC?icZ z*?uqs%g+Xl0UF4LnghCAzw8{hUt|0T`?2Uf-tmfyf@5!ea>`!VRgn{tQXu(UcS>iy z>jupytCHDYp45&;8ldf?m2^eRar#$Azq6ywRm=^2y!Yge?-X|-)9`mE{*+wGd`AZ5 zyZJd5D2B397mP)^3FDk!S*`8-o!_+0^CGn-Cie}e(Jh68)`kAWOq;hGBD{iSF5YM} zupq%mZI2f~7LWt6@~NMxM}$0tSM;2`3S4AA!*Mtqy*S_eKOh!5mPeURN_==G*nap>6gEI9wnYIgHDFwbmeJe``X%lS}5dAAp*% z%g)9=;C~%KLSdq=W1PmU_!39@?g4@a1*ILa#Hx7{=KB*{XPt+u&Yb}hy3Oh7-7Uuy z={ZL3ZTxgMKn~jp&(9GXvh;s7mhi8TtI*IkC^nr(afXAiW9F%HDwA|j+%hO{V{T}a za^vhJGd#rNURv8;CC1*|H%2C|^ijvP>Y5M{v8^{HlQC(vN|x<)n^kp?8FH|21P*zA z?vzJr)ZZ|q<8U$HTOMi>l|f*^6CnF)-PFwf$;YRi{lS7Pr*#jvl|5a^S9aQ}QL_8~ z;0xWfB4we(!o48_Q^7}4i{jY(i^lLf?(x$Y-qN*+$VC9xq<@`>Xf~c?6d_5*GETQu z{Tg`!d?92P+p+>w2+P0N=Ik$9Kz9Jt$padf(FS6>ugzl=Tt+>`O=YC{HLb@=5021$ z#;&$+>38jEUSr-|0}F^ByebBL-~ChAI&^;frxlF>W)9{clpJL>I3VPsUkBvov0X$2 z<4v080S+pV0-5fj@@Gntl;c9)OxNqcCeA$lE~L#{NJ7U|xJH6j_4DW^xgZAC7awvH zMN|)fIbI1@uu=akz=3N@KD5J;A8Tz-HWhw;}uUA%#+1z5#-8q3)UXI zX%7hfAOSH9Ooli{`YrEsuBYAvyY7JW%#ROZkUF4>M9A}*P3fM!vF-{@`cWzRnl~EQ zfz#Z^I=uyr(|m>3${oATc+M)B( zi3uC$Yg^|dJXC;y#T}?EI4ewcxbKo|ODC&_$NesI>f%g)>8BklJ2G6Noa7?T%9rbF;3JQ20kD0 z%+W4~JeDu^#7rpOjg+3+)^jUa%l=qI=np*iu!ovM!-73(jfrMl3474*l!_^pgj*M% z&|3Q>ZE5ycrOozoutO+5wNEDE;Ig0jyJMr8IKUj@;RLBm8R=2;YHAr0k< zr~1`ir<2lXR3P3R&+n_Alme1`V;JeCh$LCEPfQX$je(RLOb9EqIw-(8)S#?-^uGVg zVDe13IMH`ej$duEh!$~l(xTsBe*@OC&Pezr=M#aIG2P{iCPpGeftM+XlV0I308e=C z#oA^W%}aJz_jL>oa1tvg_9{E`CXdJ#Mmf6{G3;8b_C5YZ6%RZ-RvUSZRB<4ue2M;DhSc}`+a zLdWk(rtYHxrPpwozpFifOI9{lvtIthm3|a;vL9_8D<3hJNdy#4kHX{A2C3#{$vtmr z3Zf^ousvv+Fj3lE@7oF!no+D0NJ~v0~|M`wZ(JTtuZP>*)dFHBRNtz zpU;+-6n7o)f7{Y8(MKLE;fn&eB;LMT(Hl-kh`5RVCx1?pqzd)m18c}=tbp<&>;gJp zIkCf4jNg+cm~tDeZWUx~(X}jytV^#TIkd&Sli^unoLoy~ObSs%1-!@If~$2CP#dy3 x>t}q%-rzCO))rQJ2g$$iUu)peH!VXpi|e6wFLU9na*RIzQJ-LM_LpbA{x{NvJ-7e> literal 0 HcmV?d00001 diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index f121d67ddb1..09c1d10c32c 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -7,6 +7,8 @@ authors: tags: [LLM, research] --- +![Overall structure of AutoBuild](img/agent_autobuild.png) + **TL;DR:** Introducing **AutoBuild**, building multi-agent system automatically, fast, and easily for complex tasks with minimal user prompt required, powered by a new designed class **AgentBuilder**. AgentBuilder also support open-source LLMs by From 2e8fb36addd05d048405208a65aacca7f23a4d42 Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sat, 2 Dec 2023 02:26:08 +0800 Subject: [PATCH 58/74] try to merge --- website/docs/Examples/AgentChat.md | 57 ------------------------------ website/docs/Examples/Inference.md | 8 ----- 2 files changed, 65 deletions(-) delete mode 100644 website/docs/Examples/AgentChat.md delete mode 100644 website/docs/Examples/Inference.md diff --git a/website/docs/Examples/AgentChat.md b/website/docs/Examples/AgentChat.md deleted file mode 100644 index 629f1159b26..00000000000 --- a/website/docs/Examples/AgentChat.md +++ /dev/null @@ -1,57 +0,0 @@ -# Automated Multi Agent Chat - -AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation via multi-agent conversation. -Please find documentation about this feature [here](/docs/Use-Cases/agent_chat). - -Links to notebook examples: - - -1. **Code Generation, Execution, and Debugging** - - - Automated Task Solving with Code Generation, Execution & Debugging - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb) - - Auto Code Generation, Execution, Debugging and Human Feedback - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_human_feedback.ipynb) - - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) - - Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) - -2. **Multi-Agent Collaboration (>3 Agents)** - - - Automated Task Solving with GPT-4 + Multiple Human Users - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_two_users.ipynb) - - Automated Task Solving by Group Chat (with 3 group member agents and 1 manager agent) - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat.ipynb) - - Automated Data Visualization by Group Chat (with 3 group member agents and 1 manager agent) - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_vis.ipynb) - - Automated Complex Task Solving by Group Chat (with 6 group member agents and 1 manager agent) - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_research.ipynb) - - Automated Task Solving with Coding & Planning Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_planning.ipynb) - - Automated Task Solving with agents divided into 2 groups - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_hierarchy_flow_using_select_speaker.ipynb) - - Automated Task Solving with transition paths specified in a graph - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_graph_modelling_language_using_select_speaker.ipynb) - -3. **Applications** - - - Automated Chess Game Playing & Chitchatting by GPT-4 Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_chess.ipynb) - - Automated Continual Learning from New Data - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_stream.ipynb) - - [OptiGuide](https://github.com/microsoft/optiguide) - Coding, Tool Using, Safeguarding & Question Anwering for Supply Chain Optimization - -4. **Tool Use** - - - **Web Search**: Solve Tasks Requiring Web Info - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_web_info.ipynb) - - Use Provided Tools as Functions - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_function_call.ipynb) - - Task Solving with Langchain Provided Tools as Functions - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_function_call.ipynb) - - **RAG**: Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent) - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) - - In-depth Guide to OpenAI Utility Functions - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) - -5. **Agent Teaching and Learning** - - Teach Agents New Skills & Reuse via Automated Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teaching.ipynb) - - Teach Agents New Facts, User Preferences and Skills Beyond Coding - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachability.ipynb) - -6. **Multi-Agent Chat with OpenAI Assistants in the loop** - - Hello-World Chat with OpenAi Assistant in AutoGen - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) - - Chat with OpenAI Assistant using Function Call - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_function_call.ipynb) - - Chat with OpenAI Assistant with Code Interpreter - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_code_interpreter.ipynb) - - Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_retrieval.ipynb) - - OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_groupchat.ipynb) - -7. **Multimodal Agent** - - Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) - - Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) - - Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) - -8. **Automatic Agent Building** - - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_autobuild.ipynb) diff --git a/website/docs/Examples/Inference.md b/website/docs/Examples/Inference.md deleted file mode 100644 index ad608985ec4..00000000000 --- a/website/docs/Examples/Inference.md +++ /dev/null @@ -1,8 +0,0 @@ -# Tune GPT Models - -AutoGen also offers a cost-effective hyperparameter optimization technique [EcoOptiGen](https://arxiv.org/abs/2303.04673) for tuning Large Language Models. The research study finds that tuning hyperparameters can significantly improve the utility of them. -Please find documentation about this feature [here](/docs/Use-Cases/enhanced_inference). - -Links to notebook examples: -* [Optimize for Code Generation](https://github.com/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) -* [Optimize for Math](https://github.com/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) From e188a4d1ea56339dc5e03910982cc173dd9c5e5e Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Fri, 1 Dec 2023 10:28:35 -0800 Subject: [PATCH 59/74] Update Examples.md --- website/docs/Examples.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/docs/Examples.md b/website/docs/Examples.md index a77c48b8c57..cdaacace2f7 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -56,6 +56,8 @@ Links to notebook examples: - Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) - Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) +8. **Automatic Agent Building** + - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_autobuild.ipynb) ## Tune GPT Models From dce00563d1b0d230b521426b49be0470ea47625e Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sat, 2 Dec 2023 14:04:27 +0800 Subject: [PATCH 60/74] update test --- .github/workflows/contrib-openai.yml | 38 ++++++++++++++++++++ test/agentchat/contrib/test_agent_builder.py | 20 ----------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index f0d497b3cd6..c5d96b3f71c 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -176,3 +176,41 @@ jobs: with: file: ./coverage.xml flags: unittests + AgentBuilder: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.11" ] + runs-on: ${{ matrix.os }} + environment: openai1 + steps: + # checkout to pr branch + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies + run: | + docker --version + python -m pip install --upgrade pip wheel + pip install -e .[teachable] + python -c "import autogen" + pip install coverage pytest + - name: Coverage + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + run: | + coverage run -a -m pytest test/agentchat/contrib/test_agent_builder.py + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index 68d0d201e8a..93ee50c795e 100644 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -25,10 +25,6 @@ OPENAI_INSTALLED = False -@pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", -) def test_build(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( @@ -52,10 +48,6 @@ def test_build(): assert "TERMINATE" in agent.system_message -@pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", -) def test_save(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( @@ -85,10 +77,6 @@ def test_save(): assert saved_configs.get("default_llm_config", None) is not None -@pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", -) def test_load(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -113,10 +101,6 @@ def test_load(): assert agent_configs[agent_name]["system_message"] == agent[0].system_message -@pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", -) def test_clear_agent(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -132,10 +116,6 @@ def test_clear_agent(): assert len(builder.agent_procs_assign) == 0 -@pytest.mark.skipif( - sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED, - reason="do not run on MacOS or windows or dependency is not installed", -) def test_start(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") config_save_path = f"{here}/example_test_agent_builder_config.json" From e0bc09c633eeea748e265dbe401f2c089ecf5176 Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sat, 2 Dec 2023 14:37:37 +0800 Subject: [PATCH 61/74] skip if openai not installed --- test/agentchat/contrib/test_agent_builder.py | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index 93ee50c795e..77e3f3f9f09 100644 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -24,7 +24,10 @@ except ImportError: OPENAI_INSTALLED = False - +@pytest.mark.skipif( + not OPENAI_INSTALLED, + reason="do not run when dependency is not installed", +) def test_build(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( @@ -48,6 +51,10 @@ def test_build(): assert "TERMINATE" in agent.system_message +@pytest.mark.skipif( + not OPENAI_INSTALLED, + reason="do not run when dependency is not installed", +) def test_save(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( @@ -77,6 +84,10 @@ def test_save(): assert saved_configs.get("default_llm_config", None) is not None +@pytest.mark.skipif( + not OPENAI_INSTALLED, + reason="do not run when dependency is not installed", +) def test_load(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -101,6 +112,10 @@ def test_load(): assert agent_configs[agent_name]["system_message"] == agent[0].system_message +@pytest.mark.skipif( + not OPENAI_INSTALLED, + reason="do not run when dependency is not installed", +) def test_clear_agent(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") @@ -116,6 +131,10 @@ def test_clear_agent(): assert len(builder.agent_procs_assign) == 0 +@pytest.mark.skipif( + not OPENAI_INSTALLED, + reason="do not run when dependency is not installed", +) def test_start(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") config_save_path = f"{here}/example_test_agent_builder_config.json" From 495d7b86539812c1e5994390b016995b8ee8bf40 Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sat, 2 Dec 2023 15:27:54 +0800 Subject: [PATCH 62/74] pre-commit --- test/agentchat/contrib/test_agent_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index 77e3f3f9f09..3e6aba8a73f 100644 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -24,6 +24,7 @@ except ImportError: OPENAI_INSTALLED = False + @pytest.mark.skipif( not OPENAI_INSTALLED, reason="do not run when dependency is not installed", @@ -35,7 +36,6 @@ def test_build(): "For example, find a latest paper about gpt-4 on arxiv " "and find its potential applications in software." ) - builder.build( building_task=building_task, default_llm_config={"temperature": 0}, From e560f61a61c558b712b951fbdbf2d4f6d3392247 Mon Sep 17 00:00:00 2001 From: Linxin Song Date: Sat, 2 Dec 2023 15:38:43 +0800 Subject: [PATCH 63/74] Update website/blog/2023-11-26-Agent-AutoBuild/index.mdx Co-authored-by: Chi Wang --- website/blog/2023-11-26-Agent-AutoBuild/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index 09c1d10c32c..29fddb61852 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -136,7 +136,7 @@ After satisfying the requirements, you can add an open-source LLM's huggingface } ] ``` -and specify it when initializing AgentBuilder. , +and specify it when initializing AgentBuilder. AgentBuilder will automatically set up an endpoint server for open-source LLM. Make sure you have sufficient GPUs resources. ## Use GPTs From 1ac7e95f5164109a9093481a8f1bc1a3c993bbdc Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sun, 3 Dec 2023 02:39:49 +0800 Subject: [PATCH 64/74] update contrib-openai.yml --- .github/workflows/contrib-openai.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index c5d96b3f71c..467d5270c8e 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -197,9 +197,9 @@ jobs: run: | docker --version python -m pip install --upgrade pip wheel - pip install -e .[teachable] + pip install -e . python -c "import autogen" - pip install coverage pytest + pip install coverage pytest-asyncio - name: Coverage env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} From 260e113c695b2e5d5f08d71371d4141d311ce2ba Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sun, 3 Dec 2023 03:56:15 +0800 Subject: [PATCH 65/74] change pull_request_target to pull_request --- .github/workflows/contrib-openai.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index 467d5270c8e..e78530ac465 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -4,7 +4,7 @@ name: OpenAI4ContribTests on: - pull_request_target: + pull_request: branches: ['main'] paths: - 'autogen/**' From 89a0a707c13b3b64eead6d6a3d5aaf422d3caac6 Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sun, 3 Dec 2023 12:17:48 +0800 Subject: [PATCH 66/74] disable other openai tests --- .github/workflows/contrib-openai.yml | 326 +++++++++++++-------------- 1 file changed, 163 insertions(+), 163 deletions(-) diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index e78530ac465..890e5a30bd1 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -13,169 +13,169 @@ on: - 'setup.py' jobs: - RetrieveChatTest: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.10"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install coverage pytest-asyncio - - name: Install packages for test when needed - run: | - pip install docker - pip install qdrant_client[fastembed] - pip install -e .[retrievechat] - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - coverage run -a -m pytest test/agentchat/contrib/test_retrievechat.py test/agentchat/contrib/test_qdrant_retrievechat.py - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests - CompressionTest: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.9"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install coverage pytest-asyncio - - name: Install packages for test when needed - run: | - pip install docker - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - coverage run -a -m pytest test/agentchat/contrib/test_compressible_agent.py - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests - GPTAssistantAgent: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.11"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install coverage pytest-asyncio - - name: Install packages for test when needed - run: | - pip install docker - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - coverage run -a -m pytest test/agentchat/contrib/test_gpt_assistant.py - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests - TeachableAgent: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.11"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e .[teachable] - python -c "import autogen" - pip install coverage pytest - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - run: | - coverage run -a -m pytest test/agentchat/contrib/test_teachable_agent.py - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests +# RetrieveChatTest: +# strategy: +# matrix: +# os: [ubuntu-latest] +# python-version: ["3.10"] +# runs-on: ${{ matrix.os }} +# environment: openai1 +# steps: +# # checkout to pr branch +# - name: Checkout +# uses: actions/checkout@v3 +# with: +# ref: ${{ github.event.pull_request.head.sha }} +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install packages and dependencies +# run: | +# docker --version +# python -m pip install --upgrade pip wheel +# pip install -e . +# python -c "import autogen" +# pip install coverage pytest-asyncio +# - name: Install packages for test when needed +# run: | +# pip install docker +# pip install qdrant_client[fastembed] +# pip install -e .[retrievechat] +# - name: Coverage +# env: +# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} +# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} +# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} +# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} +# run: | +# coverage run -a -m pytest test/agentchat/contrib/test_retrievechat.py test/agentchat/contrib/test_qdrant_retrievechat.py +# coverage xml +# - name: Upload coverage to Codecov +# uses: codecov/codecov-action@v3 +# with: +# file: ./coverage.xml +# flags: unittests +# CompressionTest: +# strategy: +# matrix: +# os: [ubuntu-latest] +# python-version: ["3.9"] +# runs-on: ${{ matrix.os }} +# environment: openai1 +# steps: +# # checkout to pr branch +# - name: Checkout +# uses: actions/checkout@v3 +# with: +# ref: ${{ github.event.pull_request.head.sha }} +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install packages and dependencies +# run: | +# docker --version +# python -m pip install --upgrade pip wheel +# pip install -e . +# python -c "import autogen" +# pip install coverage pytest-asyncio +# - name: Install packages for test when needed +# run: | +# pip install docker +# - name: Coverage +# env: +# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} +# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} +# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} +# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} +# run: | +# coverage run -a -m pytest test/agentchat/contrib/test_compressible_agent.py +# coverage xml +# - name: Upload coverage to Codecov +# uses: codecov/codecov-action@v3 +# with: +# file: ./coverage.xml +# flags: unittests +# GPTAssistantAgent: +# strategy: +# matrix: +# os: [ubuntu-latest] +# python-version: ["3.11"] +# runs-on: ${{ matrix.os }} +# environment: openai1 +# steps: +# # checkout to pr branch +# - name: Checkout +# uses: actions/checkout@v3 +# with: +# ref: ${{ github.event.pull_request.head.sha }} +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install packages and dependencies +# run: | +# docker --version +# python -m pip install --upgrade pip wheel +# pip install -e . +# python -c "import autogen" +# pip install coverage pytest-asyncio +# - name: Install packages for test when needed +# run: | +# pip install docker +# - name: Coverage +# env: +# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} +# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} +# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} +# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} +# run: | +# coverage run -a -m pytest test/agentchat/contrib/test_gpt_assistant.py +# coverage xml +# - name: Upload coverage to Codecov +# uses: codecov/codecov-action@v3 +# with: +# file: ./coverage.xml +# flags: unittests +# TeachableAgent: +# strategy: +# matrix: +# os: [ubuntu-latest] +# python-version: ["3.11"] +# runs-on: ${{ matrix.os }} +# environment: openai1 +# steps: +# # checkout to pr branch +# - name: Checkout +# uses: actions/checkout@v3 +# with: +# ref: ${{ github.event.pull_request.head.sha }} +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install packages and dependencies +# run: | +# docker --version +# python -m pip install --upgrade pip wheel +# pip install -e .[teachable] +# python -c "import autogen" +# pip install coverage pytest +# - name: Coverage +# env: +# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} +# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} +# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} +# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} +# run: | +# coverage run -a -m pytest test/agentchat/contrib/test_teachable_agent.py +# coverage xml +# - name: Upload coverage to Codecov +# uses: codecov/codecov-action@v3 +# with: +# file: ./coverage.xml +# flags: unittests AgentBuilder: strategy: matrix: From 376730c8e27e1db15b040327deff49ef165c5015 Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sun, 3 Dec 2023 23:26:07 +0800 Subject: [PATCH 67/74] address issues from ekzhu; modify notebook; modify blog; modify test --- autogen/agentchat/contrib/agent_builder.py | 146 +-- notebook/agentchat_autobuild.ipynb | 1118 ++++++++++------- test/agentchat/contrib/test_agent_builder.py | 36 +- .../blog/2023-11-26-Agent-AutoBuild/index.mdx | 2 +- 4 files changed, 706 insertions(+), 596 deletions(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index 2db9b9a30f3..7e53bde32bd 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -12,9 +12,8 @@ class AgentBuilder: """ AgentBuilder can help user build an automatic task solving process powered by multi-agent system. Specifically, our building pipeline include init(), build(), and start(). - In build(), we prompt a gpt-4 model to create multiple participant agent and initialize a group chat, and specify - whether this task need programming to solve. - After that, user can call start() at a proper time to complete the task. + In build(), we prompt a gpt-4 model to create multiple participant agents, and specify whether + this task need programming to solve. User can save the built agents' config by calling save(), and load the saved configs by load(), which can skip the building process. """ @@ -79,17 +78,12 @@ def __init__( self.config_path = config_path self.endpoint_building_timeout = endpoint_building_timeout - self.user_proxy: autogen.UserProxyAgent = None - self.group_chat_manager_config: dict = None - self.initiate_agent_name: str = "User_console_and_Python_code_interpreter" - self.manager_system_message: str = "Group chat manager." - self.agent_configs: List[Dict] = [] - self.coding: bool = None - self.default_llm_config: Dict = None self.building_task: str = None + self.agent_configs: List[Dict] = [] self.open_ports: List[str] = [] self.agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} self.agent_procs_assign: Dict[str, Tuple[autogen.ConversableAgent, str]] = {} + self.cached_configs: Dict = {} for port in range(8000, 65535): if self._is_port_open(host, port): @@ -107,7 +101,7 @@ def _is_port_open(host, port): except OSError: return False - def create_agent( + def _create_agent( self, agent_name: str, model_name_or_hf_repo: str, @@ -117,7 +111,7 @@ def create_agent( world_size: Optional[int] = 1, ) -> autogen.AssistantAgent: """ - Create a group chat agent. + Create group chat agents. If the agent rely on an open-source model, this function will automatically set up an endpoint for that agent. The API address of that endpoint will be "localhost:{free port}". @@ -269,16 +263,13 @@ def build( if cached_configs is None: use_api = True + agent_configs = [] self.building_task = building_task - self.default_llm_config = default_llm_config.copy() - self.coding = coding - self.agent_configs = [] else: - self.building_task = cached_configs["building_task"] - self.default_llm_config = cached_configs["default_llm_config"] - self.coding = cached_configs["coding"] - self.agent_configs = cached_configs["agent_configs"] - self.manager_system_message = cached_configs["manager_system_message"] + self.building_task = building_task = cached_configs["building_task"] + default_llm_config = cached_configs["default_llm_config"] + coding = cached_configs["coding"] + agent_configs = cached_configs["agent_configs"] config_list = autogen.config_list_from_json(self.config_path, filter_dict={"model": [self.builder_model]}) build_manager = autogen.OpenAIWrapper(config_list=config_list) @@ -290,9 +281,7 @@ def build( messages=[ { "role": "user", - "content": self.AGENT_NAME_PROMPT.format( - task=self.building_task, max_agents=self.max_agents - ), + "content": self.AGENT_NAME_PROMPT.format(task=building_task, max_agents=self.max_agents), } ] ) @@ -311,7 +300,7 @@ def build( { "role": "user", "content": self.AGENT_SYS_MSG_PROMPT.format( - task=self.building_task, + task=building_task, position=name, default_sys_msg=autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, ), @@ -324,51 +313,59 @@ def build( agent_sys_msg_list.append(resp_agent_sys_msg) for i in range(len(agent_name_list)): - self.agent_configs.append( + agent_configs.append( {"name": agent_name_list[i], "model": self.agent_model, "system_message": agent_sys_msg_list[i]} ) - self.manager_system_message = "Group chat manager." - for config in self.agent_configs: + for config in agent_configs: print(f"Creating agent {config['name']} with backbone {config['model']}...") - self.create_agent( + self._create_agent( config["name"], config["model"], - self.default_llm_config, + default_llm_config, system_message=config["system_message"], use_gpts=use_gpts, **kwargs, ) + agent_list = [agent_config[0] for agent_config in self.agent_procs_assign.values()] - if self.coding is None: + if coding is None: resp = ( build_manager.create( - messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=self.building_task)}] + messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=building_task)}] ) .choices[0] .message.content ) - self.coding = True if resp == "YES" else False + coding = True if resp == "YES" else False - if self.coding is True: + if coding is True: print("Adding user console proxy...") - self.user_proxy = autogen.UserProxyAgent( - name="User_console_and_Python_code_interpreter", - is_termination_msg=lambda x: "TERMINATE" in x.get("content"), - system_message="User console with a python code interpreter interface.", - code_execution_config={ - "last_n_messages": 2, - "work_dir": user_proxy_work_dir, - "use_docker": docker, - "timeout": 60, - }, - human_input_mode="NEVER", - ) - else: - self.initiate_agent_name = self.agent_configs[0]["name"] + agent_list = [ + autogen.UserProxyAgent( + name="User_console_and_Python_code_interpreter", + is_termination_msg=lambda x: "TERMINATE" in x.get("content"), + system_message="User console with a python code interpreter interface.", + code_execution_config={ + "last_n_messages": 2, + "work_dir": user_proxy_work_dir, + "use_docker": docker, + "timeout": 60, + }, + human_input_mode="NEVER", + ) + ] + agent_list - self.group_chat_manager_config = self.default_llm_config.copy() - self.group_chat_manager_config["config_list"] = config_list + self.cached_configs.update( + { + "building_task": building_task, + "agent_configs": agent_configs, + "coding": coding, + "default_llm_config": default_llm_config, + } + ) + + return agent_list, self.cached_configs.copy() def save(self, filepath: Optional[str] = None) -> str: """ @@ -383,17 +380,8 @@ def save(self, filepath: Optional[str] = None) -> str: """ if filepath is None: filepath = f'./save_config_{hashlib.md5(self.building_task.encode("utf-8")).hexdigest()}.json' - json.dump( - { - "building_task": self.building_task, - "agent_configs": self.agent_configs, - "manager_system_message": self.manager_system_message, - "coding": self.coding, - "default_llm_config": self.default_llm_config, - }, - open(filepath, "w"), - indent=4, - ) + with open(filepath, "w") as save_file: + json.dump(self.cached_configs, save_file, indent=4) print(f"Building config saved to {filepath}") return filepath @@ -409,42 +397,10 @@ def load( Args: filepath: filepath for the save config. """ - if os.path.isfile(filepath): + try: print(f"Loding config from {filepath}") cached_configs = json.load(open(filepath)) - self.build(cached_configs=cached_configs, **kwargs) - else: + except FileNotFoundError: raise FileNotFoundError(f"Config file {filepath} does not exist.") - return self - - def start( - self, task: str, max_round: Optional[int] = 12, init_messages: Optional[List[dict]] = [] - ) -> Tuple[autogen.GroupChat, autogen.GroupChatManager]: - """ - Start a group chat task solving process with built config. - - Args: - task: description of a task. - max_round: the maximum number of rounds. - init_messages: input messages before the task start. This can be the chat history from other group chat - or some preliminary of the task. - """ - agent_list = [agent for agent, _ in self.agent_procs_assign.values()] - if self.user_proxy is not None: - agent_list.append(self.user_proxy) - # TODO: add agent trigger - group_chat = autogen.GroupChat(agents=agent_list, messages=init_messages, max_round=max_round) - - manager = autogen.GroupChatManager( - groupchat=group_chat, llm_config=self.group_chat_manager_config, system_message=self.manager_system_message - ) - - if self.initiate_agent_name == "User_console_and_Python_code_interpreter" and self.user_proxy is not None: - self.user_proxy.initiate_chat(manager, message=task) - else: - for agent in agent_list: - if self.initiate_agent_name == agent.name: - agent.initiate_chat(manager, message=task) - - return group_chat, manager + return self.build(cached_configs=cached_configs, **kwargs) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index 5b5768dcb97..fab66c2ee59 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -52,18 +52,18 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 6, "id": "2505f029423b21ab", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:18.987795800Z", - "start_time": "2023-11-28T09:23:18.977219Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:31:44.147211100Z", + "start_time": "2023-12-03T13:31:44.121842300Z" + } }, "outputs": [], "source": [ - "config_path = '/home/elpis_ubuntu/LLM/autogen/OAI_CONFIG_LIST' # modify path\n", + "config_path = 'OAI_CONFIG_LIST' # modify path\n", "default_llm_config = {\n", " 'temperature': 0\n", "}" @@ -82,14 +82,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 7, "id": "bfa67c771a0fed37", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:19.902186400Z", - "start_time": "2023-11-28T09:23:18.979228600Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:31:44.996307300Z", + "start_time": "2023-12-03T13:31:44.743284700Z" + } }, "outputs": [], "source": [ @@ -112,18 +112,18 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "id": "68315f6ec912c58a", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:21.419250500Z", - "start_time": "2023-11-28T09:23:21.413220200Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:31:45.444044500Z", + "start_time": "2023-12-03T13:31:45.429483200Z" + } }, "outputs": [], "source": [ - "building_task = \"Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"" + "building_task = \"Find a paper on arxiv by programming, and analyze its application in some domain. For example, find a recent paper about gpt-4 on arxiv and find its potential applications in software.\"" ] }, { @@ -143,14 +143,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "id": "ab490fdbe46c0473", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:23.564327400Z", - "start_time": "2023-11-28T09:23:23.488457300Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:32:45.887656900Z", + "start_time": "2023-12-03T13:31:46.822373400Z" + } }, "outputs": [ { @@ -158,19 +158,19 @@ "output_type": "stream", "text": [ "Generating agents...\n", - "Data_scientist,Domain_expert,Software_engineer are generated.\n", + "Data_scientist,Research_analyst,Software_developer are generated.\n", "Preparing configuration for Data_scientist...\n", - "Preparing configuration for Domain_expert...\n", - "Preparing configuration for Software_engineer...\n", + "Preparing configuration for Research_analyst...\n", + "Preparing configuration for Software_developer...\n", "Creating agent Data_scientist with backbone gpt-4-1106-preview...\n", - "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", - "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", + "Creating agent Research_analyst with backbone gpt-4-1106-preview...\n", + "Creating agent Software_developer with backbone gpt-4-1106-preview...\n", "Adding user console proxy...\n" ] } ], "source": [ - "builder.build(building_task, default_llm_config)" + "agent_list, agent_configs = builder.build(building_task, default_llm_config)" ] }, { @@ -186,207 +186,203 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "id": "7d52e3d9a1bf91cb", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:27.331284800Z", - "start_time": "2023-11-28T09:23:25.264410200Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:33:42.369660600Z", + "start_time": "2023-12-03T13:32:45.881740500Z" + } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", - "\n", - "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "Find a recent paper about gpt-4 on arxiv and find its potential applications in software.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", - "\n", - "To find the latest paper about GPT-4 on arXiv, we can use the arXiv API to search for papers that match the query. We'll write a Python script that uses the `requests` library to make a GET request to the arXiv API with the appropriate query parameters. The script will parse the response to find the most recent paper related to GPT-4 and then print out the title, authors, summary, and potential applications in software.\n", + "\u001B[33mData_scientist\u001B[0m (to chat_manager):\n", "\n", - "First, ensure you have the `requests` library installed. If not, you can install it using pip:\n", + "To begin, we'll write a Python script that uses the `arxiv` library to search for recent papers on arXiv related to GPT-4. The script will automate the process of searching for the papers, downloading the metadata, and then extracting the relevant information to identify potential applications in software.\n", "\n", - "```bash\n", - "pip install requests\n", - "```\n", + "First, you'll need to install the `arxiv` library if you haven't already. You can do this by running `pip install arxiv`.\n", "\n", - "Now, here's the Python script to find the latest paper about GPT-4 on arXiv:\n", + "Here's a Python script that will perform the search and print out the title, authors, summary, and publication date of the most recent papers related to GPT-4. Save this script to a file and run it in your Python environment.\n", "\n", "```python\n", - "import requests\n", - "from datetime import datetime\n", - "import xml.etree.ElementTree as ET\n", - "\n", - "# Function to search arXiv for papers related to GPT-4\n", - "def search_arxiv(query):\n", - " # Define the URL and parameters for the arXiv API request\n", - " url = 'http://export.arxiv.org/api/query'\n", - " params = {\n", - " 'search_query': query,\n", - " 'sortBy': 'submittedDate',\n", - " 'sortOrder': 'descending',\n", - " 'max_results': 1\n", - " }\n", - " \n", - " # Make the GET request to the arXiv API\n", - " response = requests.get(url, params=params)\n", - " \n", - " # Check if the request was successful\n", - " if response.status_code == 200:\n", - " return response.text\n", - " else:\n", - " print(f\"Error: {response.status_code}\")\n", - " return None\n", - "\n", - "# Function to parse the XML response and extract paper details\n", - "def parse_arxiv_response(xml_response):\n", - " # Parse the XML response\n", - " root = ET.fromstring(xml_response)\n", - " \n", - " # Find the entry element, which contains the paper information\n", - " entry = root.find('{http://www.w3.org/2005/Atom}entry')\n", - " \n", - " # Extract the title, authors, summary, and published date\n", - " title = entry.find('{http://www.w3.org/2005/Atom}title').text.strip()\n", - " authors = [author.find('{http://www.w3.org/2005/Atom}name').text for author in entry.findall('{http://www.w3.org/2005/Atom}author')]\n", - " summary = entry.find('{http://www.w3.org/2005/Atom}summary').text.strip()\n", - " published_date = entry.find('{http://www.w3.org/2005/Atom}published').text\n", - " \n", - " return {\n", - " 'title': title,\n", - " 'authors': authors,\n", - " 'summary': summary,\n", - " 'published_date': published_date\n", - " }\n", - "\n", - "# Main function to execute the script\n", - "def main():\n", - " # Search for the latest paper on GPT-4\n", - " query = 'all:gpt-4'\n", - " xml_response = search_arxiv(query)\n", - " \n", - " if xml_response:\n", - " # Parse the response and extract paper details\n", - " paper_details = parse_arxiv_response(xml_response)\n", - " \n", - " # Print the paper details\n", - " print(f\"Title: {paper_details['title']}\")\n", - " print(f\"Authors: {', '.join(paper_details['authors'])}\")\n", - " print(f\"Summary: {paper_details['summary']}\")\n", - " print(f\"Published Date: {paper_details['published_date']}\")\n", - " \n", - " # Analyze potential applications in software based on the summary\n", - " # This part requires human analysis and cannot be automated\n", - " print(\"\\nPotential applications in software:\")\n", - " print(\"The potential applications in software based on the summary of the paper would require a detailed analysis and understanding of the content. This step should be performed by a domain expert who can interpret the findings and suggest practical applications.\")\n", - "\n", - "if __name__ == '__main__':\n", - " main()\n", - "```\n", + "# filename: arxiv_search_gpt4.py\n", "\n", - "This script will print the details of the latest paper about GPT-4 on arXiv. After running the script, you should manually analyze the summary to evaluate the potential applications in software. This analysis requires domain expertise and cannot be fully automated.\n", + "import arxiv\n", + "import datetime\n", "\n", - "Please run the script, and I will assist you with the analysis of the potential applications in software based on the information retrieved.\n", + "# Define the search query and parameters\n", + "search_query = 'all:\"GPT-4\"'\n", + "max_results = 5 # You can adjust this number based on how many results you want\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "# Search arXiv for papers related to GPT-4\n", + "search = arxiv.Search(\n", + " query=search_query,\n", + " max_results=max_results,\n", + " sort_by=arxiv.SortCriterion.SubmittedDate\n", + ")\n", "\n", - "exitcode: 0 (execution succeeded)\n", - "Code output: \n", - "Requirement already satisfied: requests in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.31.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (1.26.16)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests) (2023.7.22)\n", - "\n", - "Title: MEDITRON-70B: Scaling Medical Pretraining for Large Language Models\n", - "Authors: Zeming Chen, Alejandro Hernández Cano, Angelika Romanou, Antoine Bonnet, Kyle Matoba, Francesco Salvi, Matteo Pagliardini, Simin Fan, Andreas Köpf, Amirkeivan Mohtashami, Alexandre Sallinen, Alireza Sakhaeirad, Vinitra Swamy, Igor Krawczuk, Deniz Bayazit, Axel Marmet, Syrielle Montariol, Mary-Anne Hartley, Martin Jaggi, Antoine Bosselut\n", - "Summary: Large language models (LLMs) can potentially democratize access to medical\n", - "knowledge. While many efforts have been made to harness and improve LLMs'\n", - "medical knowledge and reasoning capacities, the resulting models are either\n", - "closed-source (e.g., PaLM, GPT-4) or limited in scale (<= 13B parameters),\n", - "which restricts their abilities. In this work, we improve access to large-scale\n", - "medical LLMs by releasing MEDITRON: a suite of open-source LLMs with 7B and 70B\n", - "parameters adapted to the medical domain. MEDITRON builds on Llama-2 (through\n", - "our adaptation of Nvidia's Megatron-LM distributed trainer), and extends\n", - "pretraining on a comprehensively curated medical corpus, including selected\n", - "PubMed articles, abstracts, and internationally-recognized medical guidelines.\n", - "Evaluations using four major medical benchmarks show significant performance\n", - "gains over several state-of-the-art baselines before and after task-specific\n", - "finetuning. Overall, MEDITRON achieves a 6% absolute performance gain over the\n", - "best public baseline in its parameter class and 3% over the strongest baseline\n", - "we finetuned from Llama-2. Compared to closed-source LLMs, MEDITRON-70B\n", - "outperforms GPT-3.5 and Med-PaLM and is within 5% of GPT-4 and 10% of\n", - "Med-PaLM-2. We release our code for curating the medical pretraining corpus and\n", - "the MEDITRON model weights to drive open-source development of more capable\n", - "medical LLMs.\n", - "Published Date: 2023-11-27T18:49:43Z\n", - "\n", - "Potential applications in software:\n", - "The potential applications in software based on the summary of the paper would require a detailed analysis and understanding of the content. This step should be performed by a domain expert who can interpret the findings and suggest practical applications.\n", + "# Fetch the results\n", + "results = list(search.results())\n", "\n", + "# Print the details of the most recent papers\n", + "for result in results:\n", + " published = result.published.strftime('%Y-%m-%d')\n", + " print(f\"Title: {result.title}\\nAuthors: {', '.join(author.name for author in result.authors)}\\nPublished: {published}\\nSummary: {result.summary}\\n\")\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[33mDomain_expert\u001b[0m (to chat_manager):\n", + "# Note: This script does not download the full paper, only the metadata.\n", + "```\n", "\n", - "Based on the summary of the paper titled \"MEDITRON-70B: Scaling Medical Pretraining for Large Language Models,\" we can infer several potential applications in software, particularly within the medical domain:\n", + "After running this script, you will have a list of recent papers related to GPT-4. You can then read through the summaries to identify potential applications in software. If you need to download the full papers, you can modify the script to fetch the PDFs using the URLs provided in the metadata.\n", "\n", - "1. **Medical Knowledge Access**: MEDITRON-70B, as an open-source large language model (LLM) adapted to the medical domain, can be integrated into software platforms to provide healthcare professionals and researchers with easier access to medical knowledge. This could be in the form of intelligent search engines or decision support systems that can understand and process complex medical queries.\n", + "Once you have the summaries or full papers, you can use your analytical skills to discern the potential applications of GPT-4 in software. Look for keywords such as \"software engineering\", \"application\", \"tool\", \"framework\", \"integration\", \"development\", and \"automation\" to find relevant information.\n", "\n", - "2. **Clinical Decision Support**: The model's ability to understand and process medical literature can be leveraged to develop clinical decision support tools that assist healthcare providers in diagnosing and treating patients. By integrating MEDITRON-70B into electronic health record (EHR) systems, it could provide real-time, evidence-based recommendations.\n", + "Please execute the above script to retrieve the recent papers on GPT-4 from arXiv. After that, I can guide you through the analysis of their content to identify potential applications in software.\n", "\n", - "3. **Medical Education**: Educational software can utilize MEDITRON-70B to create interactive learning experiences for medical students and professionals. The model can generate quizzes, explain complex medical concepts, and simulate patient interactions for training purposes.\n", + "--------------------------------------------------------------------------------\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", - "4. **Research Acceleration**: Researchers can use MEDITRON-70B to quickly sift through vast amounts of medical literature to find relevant studies, thus accelerating the pace of medical research. This could be particularly useful in systematic reviews and meta-analyses.\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Title: Unnatural Error Correction: GPT-4 Can Almost Perfectly Handle Unnatural Scrambled Text\n", + "Authors: Qi Cao, Takeshi Kojima, Yutaka Matsuo, Yusuke Iwasawa\n", + "Published: 2023-11-30\n", + "Summary: While Large Language Models (LLMs) have achieved remarkable performance in\n", + "many tasks, much about their inner workings remains unclear. In this study, we\n", + "present novel experimental insights into the resilience of LLMs, particularly\n", + "GPT-4, when subjected to extensive character-level permutations. To investigate\n", + "this, we first propose the Scrambled Bench, a suite designed to measure the\n", + "capacity of LLMs to handle scrambled input, in terms of both recovering\n", + "scrambled sentences and answering questions given scrambled context. The\n", + "experimental results indicate that most powerful LLMs demonstrate the\n", + "capability akin to typoglycemia, a phenomenon where humans can understand the\n", + "meaning of words even when the letters within those words are scrambled, as\n", + "long as the first and last letters remain in place. More surprisingly, we found\n", + "that only GPT-4 nearly flawlessly processes inputs with unnatural errors, even\n", + "under the extreme condition, a task that poses significant challenges for other\n", + "LLMs and often even for humans. Specifically, GPT-4 can almost perfectly\n", + "reconstruct the original sentences from scrambled ones, decreasing the edit\n", + "distance by 95%, even when all letters within each word are entirely scrambled.\n", + "It is counter-intuitive that LLMs can exhibit such resilience despite severe\n", + "disruption to input tokenization caused by scrambled text.\n", + "\n", + "Title: Language Model Agents Suffer from Compositional Generalization in Web Automation\n", + "Authors: Hiroki Furuta, Yutaka Matsuo, Aleksandra Faust, Izzeddin Gur\n", + "Published: 2023-11-30\n", + "Summary: Language model agents (LMA) recently emerged as a promising paradigm on\n", + "muti-step decision making tasks, often outperforming humans and other\n", + "reinforcement learning agents. Despite the promise, their performance on\n", + "real-world applications that often involve combinations of tasks is still\n", + "underexplored. In this work, we introduce a new benchmark, called CompWoB -- 50\n", + "new compositional web automation tasks reflecting more realistic assumptions.\n", + "We show that while existing prompted LMAs (gpt-3.5-turbo or gpt-4) achieve\n", + "94.0% average success rate on base tasks, their performance degrades to 24.9%\n", + "success rate on compositional tasks. On the other hand, transferred LMAs\n", + "(finetuned only on base tasks) show less generalization gap, dropping from\n", + "85.4% to 54.8%. By balancing data distribution across tasks, we train a new\n", + "model, HTML-T5++, that surpasses human-level performance (95.2%) on MiniWoB,\n", + "and achieves the best zero-shot performance on CompWoB (61.5%). While these\n", + "highlight the promise of small-scale finetuned and transferred models for\n", + "compositional generalization, their performance further degrades under\n", + "different instruction compositions changing combinational order. In contrast to\n", + "the recent remarkable success of LMA, our benchmark and detailed analysis\n", + "emphasize the necessity of building LMAs that are robust and generalizable to\n", + "task compositionality for real-world deployment.\n", + "\n", + "Title: AlignBench: Benchmarking Chinese Alignment of Large Language Models\n", + "Authors: Xiao Liu, Xuanyu Lei, Shengyuan Wang, Yue Huang, Zhuoer Feng, Bosi Wen, Jiale Cheng, Pei Ke, Yifan Xu, Weng Lam Tam, Xiaohan Zhang, Lichao Sun, Hongning Wang, Jing Zhang, Minlie Huang, Yuxiao Dong, Jie Tang\n", + "Published: 2023-11-30\n", + "Summary: Alignment has become a critical step for instruction-tuned Large Language\n", + "Models (LLMs) to become helpful assistants. However, effective evaluation of\n", + "alignment for emerging Chinese LLMs is still significantly lacking, calling for\n", + "real-scenario grounded, open-ended, challenging and automatic evaluations\n", + "tailored for alignment. To fill in this gap, we introduce AlignBench, a\n", + "comprehensive multi-dimensional benchmark for evaluating LLMs' alignment in\n", + "Chinese. Equipped with a human-in-the-loop data curation pipeline, our\n", + "benchmark employs a rule-calibrated multi-dimensional LLM-as-Judge with\n", + "Chain-of-Thought to generate explanations and final ratings as evaluations,\n", + "ensuring high reliability and interpretability. Furthermore, we developed a\n", + "dedicated companion evaluator LLM -- CritiqueLLM, which recovers 95\\% of\n", + "GPT-4's evaluation ability and will be provided via public APIs to researchers\n", + "for evaluation of alignment in Chinese LLMs. All evaluation codes, data, and\n", + "LLM generations are available at \\url{https://github.com/THUDM/AlignBench}.\n", + "\n", + "Title: CritiqueLLM: Scaling LLM-as-Critic for Effective and Explainable Evaluation of Large Language Model Generation\n", + "Authors: Pei Ke, Bosi Wen, Zhuoer Feng, Xiao Liu, Xuanyu Lei, Jiale Cheng, Shengyuan Wang, Aohan Zeng, Yuxiao Dong, Hongning Wang, Jie Tang, Minlie Huang\n", + "Published: 2023-11-30\n", + "Summary: Since the natural language processing (NLP) community started to make large\n", + "language models (LLMs), such as GPT-4, act as a critic to evaluate the quality\n", + "of generated texts, most of them only train a critique generation model of a\n", + "specific scale on specific datasets. We argue that a comprehensive\n", + "investigation on the key factor of LLM-based evaluation models, such as scaling\n", + "properties, is lacking, so that it is still inconclusive whether these models\n", + "have potential to replace GPT-4's evaluation in practical scenarios. In this\n", + "paper, we propose a new critique generation model called CritiqueLLM, which\n", + "includes a dialogue-based prompting method for high-quality referenced /\n", + "reference-free evaluation data. Experimental results show that our model can\n", + "achieve comparable evaluation performance to GPT-4 especially in system-level\n", + "correlations, and even outperform GPT-4 in 3 out of 8 tasks in a challenging\n", + "reference-free setting. We conduct detailed analysis to show promising scaling\n", + "properties of our model in the quality of generated critiques. We also\n", + "demonstrate that our generated critiques can act as scalable feedback to\n", + "directly improve the generation quality of LLMs.\n", + "\n", + "Title: AviationGPT: A Large Language Model for the Aviation Domain\n", + "Authors: Liya Wang, Jason Chou, Xin Zhou, Alex Tien, Diane M Baumgartner\n", + "Published: 2023-11-29\n", + "Summary: The advent of ChatGPT and GPT-4 has captivated the world with large language\n", + "models (LLMs), demonstrating exceptional performance in question-answering,\n", + "summarization, and content generation. The aviation industry is characterized\n", + "by an abundance of complex, unstructured text data, replete with technical\n", + "jargon and specialized terminology. Moreover, labeled data for model building\n", + "are scarce in this domain, resulting in low usage of aviation text data. The\n", + "emergence of LLMs presents an opportunity to transform this situation, but\n", + "there is a lack of LLMs specifically designed for the aviation domain. To\n", + "address this gap, we propose AviationGPT, which is built on open-source LLaMA-2\n", + "and Mistral architectures and continuously trained on a wealth of carefully\n", + "curated aviation datasets. Experimental results reveal that AviationGPT offers\n", + "users multiple advantages, including the versatility to tackle diverse natural\n", + "language processing (NLP) problems (e.g., question-answering, summarization,\n", + "document writing, information extraction, report querying, data cleaning, and\n", + "interactive data exploration). It also provides accurate and contextually\n", + "relevant responses within the aviation domain and significantly improves\n", + "performance (e.g., over a 40% performance gain in tested cases). With\n", + "AviationGPT, the aviation industry is better equipped to address more complex\n", + "research problems and enhance the efficiency and safety of National Airspace\n", + "System (NAS) operations.\n", "\n", - "5. **Telemedicine**: In telemedicine applications, MEDITRON-70B could be used to pre-assess patient symptoms and provide preliminary advice, helping to triage cases and manage healthcare resources more effectively.\n", "\n", - "6. **Language Translation and Localization**: Given the model's large-scale pretraining on medical texts, it could be used to translate medical documents and guidelines into different languages, making medical knowledge more accessible globally.\n", "\n", - "7. **Personalized Medicine**: Software applications could use MEDITRON-70B to analyze patient data and generate personalized treatment plans based on the latest medical research and guidelines.\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mData_scientist\u001B[0m (to chat_manager):\n", "\n", - "8. **Healthcare Chatbots**: The model could power chatbots that provide medical information to patients, answer frequently asked questions, and guide them through administrative processes like appointment scheduling.\n", + "Based on the output, we have several recent papers related to GPT-4. Let's analyze their summaries to identify potential applications in software:\n", "\n", - "9. **Drug Discovery and Development**: MEDITRON-70B could assist in the drug discovery process by analyzing scientific literature to identify potential drug targets and predict the effects of drug compounds.\n", + "1. **Unnatural Error Correction: GPT-4 Can Almost Perfectly Handle Unnatural Scrambled Text**\n", + " - **Potential Application**: This paper suggests that GPT-4 has a remarkable ability to correct and understand scrambled text. This could be applied in software for error correction, data cleaning, and improving resilience against data corruption or obfuscation.\n", "\n", - "10. **Public Health Monitoring**: The model could be used to monitor public health trends by analyzing social media, news articles, and other text sources for mentions of disease outbreaks or health-related events.\n", + "2. **Language Model Agents Suffer from Compositional Generalization in Web Automation**\n", + " - **Potential Application**: The paper discusses the performance of GPT-4 in web automation tasks and highlights its limitations in compositional generalization. This indicates that while GPT-4 can be used in web automation software, there is room for improvement, especially in tasks that require understanding and combining different instructions.\n", "\n", - "It's important to note that while these applications are promising, they also require careful consideration of ethical implications, data privacy, and the need for rigorous validation to ensure patient safety and the accuracy of medical information provided by the software.\n", + "3. **AlignBench: Benchmarking Chinese Alignment of Large Language Models**\n", + " - **Potential Application**: This paper introduces a benchmark for evaluating the alignment of Chinese LLMs, including GPT-4. The potential application here is in developing software tools for evaluating and improving the alignment of language models, particularly for non-English languages, which is crucial for creating more inclusive and effective NLP applications.\n", + "\n", + "4. **CritiqueLLM: Scaling LLM-as-Critic for Effective and Explainable Evaluation of Large Language Model Generation**\n", + " - **Potential Application**: The research presents a model for evaluating the quality of text generated by LLMs. Software applications could include automated quality control for content generation, providing feedback for improving language models, and developing more explainable AI systems.\n", + "\n", + "5. **AviationGPT: A Large Language Model for the Aviation Domain**\n", + " - **Potential Application**: The paper proposes a domain-specific LLM for aviation. This model could be applied in software for various NLP tasks within the aviation industry, such as question-answering, summarization, document writing, information extraction, report querying, data cleaning, and interactive data exploration, leading to improved efficiency and safety in aviation operations.\n", + "\n", + "These summaries provide a glimpse into the diverse applications of GPT-4 in software. From error correction and web automation to domain-specific applications and model evaluation, GPT-4's capabilities can be leveraged to enhance various aspects of software development and deployment.\n", "\n", "TERMINATE\n", "\n", @@ -395,7 +391,22 @@ } ], "source": [ - "builder.start(task=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\")" + "import autogen\n", + "\n", + "def start_task(execution_task: str, agent_list: list, llm_config: dict):\n", + " config_list = autogen.config_list_from_json(config_path, filter_dict={\"model\": [\"gpt-4-1106-preview\"]})\n", + " \n", + " group_chat = autogen.GroupChat(agents=agent_list, messages=[], max_round=12)\n", + " manager = autogen.GroupChatManager(\n", + " groupchat=group_chat, llm_config={\"config_list\": config_list, **llm_config}\n", + " )\n", + " agent_list[0].initiate_chat(manager, message=execution_task)\n", + "\n", + "start_task(\n", + " execution_task=\"Find a recent paper about gpt-4 on arxiv and find its potential applications in software.\",\n", + " agent_list=agent_list,\n", + " llm_config=default_llm_config\n", + ")" ] }, { @@ -411,14 +422,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, "id": "7fb0bfff01dd1330", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:28.687772900Z", - "start_time": "2023-11-28T09:23:28.681728600Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:34:08.429248500Z", + "start_time": "2023-12-03T13:34:08.364799400Z" + } }, "outputs": [ { @@ -475,21 +486,21 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 12, "id": "e4b88a5d482ceba4", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:29.950261800Z", - "start_time": "2023-11-28T09:23:29.944238100Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:34:09.652503400Z", + "start_time": "2023-12-03T13:34:09.639760500Z" + } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Building config saved to ./save_config_eb1be857faa608aeb4c5af11fe4ab245.json\n" + "Building config saved to ./save_config_2a87f83309bd8b5ebb0423f5570385f5.json\n" ] } ], @@ -509,125 +520,127 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 13, "id": "34addd498e5ab174", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:23:39.363905700Z", - "start_time": "2023-11-28T09:23:31.493375Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:34:56.857918300Z", + "start_time": "2023-12-03T13:34:11.108958800Z" + } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Loding config from ./save_config_eb1be857faa608aeb4c5af11fe4ab245.json\n", + "Loding config from ./save_config_2a87f83309bd8b5ebb0423f5570385f5.json\n", "Creating agent Data_scientist with backbone gpt-4-1106-preview...\n", - "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", - "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", + "Creating agent Research_analyst with backbone gpt-4-1106-preview...\n", + "Creating agent Software_developer with backbone gpt-4-1106-preview...\n", "Adding user console proxy...\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", - "Find a latest paper about Llava on arxiv and find its potential applications in multi-modal model.\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "Find a recent paper about Llava on arxiv and find its potential applications in computer vision.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", - "\n", - "To find the latest paper about \"Llava\" on arXiv and evaluate its potential applications in multi-modal models, we will break down the task into the following steps:\n", + "\u001B[33mData_scientist\u001B[0m (to chat_manager):\n", "\n", - "1. Search for the latest papers on arXiv with the keyword \"Llava\".\n", - "2. Download and read the most recent paper found.\n", - "3. Analyze the content to determine its potential applications in multi-modal models.\n", + "To find a recent paper about \"Llava\" on arXiv, we can use the arXiv API to search for papers that match this keyword. However, it's important to note that \"Llava\" might be a typo or a less common term. If you meant \"Lava\" or another term, please correct me. Assuming \"Llava\" is the correct term, I will proceed with that.\n", "\n", - "Let's start with step 1. We will use the `arxiv` Python library to search for papers. If you don't have the `arxiv` library installed, you can install it using pip:\n", - "\n", - "```bash\n", - "pip install arxiv\n", - "```\n", - "\n", - "Now, let's write a Python script to search for the latest papers on arXiv with the keyword \"Llava\":\n", + "Here's a Python script that uses the `arxiv` library to search for papers related to \"Llava\". If the `arxiv` library is not installed on your system, you can install it using `pip install arxiv`.\n", "\n", "```python\n", + "# filename: arxiv_search.py\n", + "\n", "import arxiv\n", - "import datetime\n", "\n", "# Define the search query and parameters\n", "search_query = 'all:Llava'\n", - "sort_by = arxiv.SortCriterion.SubmittedDate\n", - "sort_order = arxiv.SortOrder.Descending\n", - "\n", - "# Search for papers on arXiv\n", - "search = arxiv.Search(\n", - " query=search_query,\n", - " max_results=1,\n", - " sort_by=sort_by,\n", - " sort_order=sort_order\n", - ")\n", - "\n", - "# Fetch the result\n", - "try:\n", - " paper = next(search.results())\n", - " print(f\"Title: {paper.title}\")\n", - " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", - " print(f\"Abstract: {paper.summary}\")\n", - " print(f\"Published: {paper.published}\")\n", - " print(f\"URL: {paper.entry_id}\")\n", - "except StopIteration:\n", - " print(\"No papers found with the keyword 'Llava'.\")\n", - "except Exception as e:\n", - " print(f\"An error occurred: {e}\")\n", + "max_results = 10\n", + "\n", + "# Search arXiv for papers related to the search query\n", + "def search_papers(query, max_results):\n", + " search = arxiv.Search(\n", + " query=query,\n", + " max_results=max_results,\n", + " sort_by=arxiv.SortCriterion.SubmittedDate\n", + " )\n", + " for result in search.results():\n", + " print(f\"Title: {result.title}\")\n", + " print(f\"Authors: {', '.join(author.name for author in result.authors)}\")\n", + " print(f\"Abstract: {result.summary}\")\n", + " print(f\"URL: {result.entry_id}\")\n", + " print(f\"Published: {result.published}\")\n", + " print(\"\")\n", + "\n", + "# Run the search and print the results\n", + "search_papers(search_query, max_results)\n", "```\n", "\n", - "This script will search for the most recent papers on arXiv with the keyword \"Llava\" and print out the title, authors, abstract, publication date, and URL of the latest paper.\n", - "\n", - "Please run this script in your Python environment to find the latest paper. Once we have the paper, we can proceed to step 2 and 3 to download, read, and analyze its content for potential applications in multi-modal models.\n", + "To execute this script, save it to a file named `arxiv_search.py` and run it using a Python interpreter. The script will print out the titles, authors, abstracts, and URLs of up to 10 recent papers related to \"Llava\".\n", "\n", - "After running the script, please provide the title and abstract of the paper so that we can discuss its potential applications.\n", + "Once we have the papers, we can analyze their abstracts to determine potential applications in computer vision. However, this part of the task will require human analysis and cannot be fully automated, as it involves understanding and interpreting the content of the papers. If the script finds relevant papers, I will proceed with the analysis based on the abstracts provided.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is bash)...\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Requirement already satisfied: arxiv in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (2.0.0)\n", - "Requirement already satisfied: feedparser==6.0.10 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from arxiv) (6.0.10)\n", - "Requirement already satisfied: requests==2.31.0 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from arxiv) (2.31.0)\n", - "Requirement already satisfied: sgmllib3k in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from feedparser==6.0.10->arxiv) (1.0.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (1.26.16)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.10/site-packages (from requests==2.31.0->arxiv) (2023.7.22)\n", + "Title: Contrastive Vision-Language Alignment Makes Efficient Instruction Learner\n", + "Authors: Lizhao Liu, Xinyu Sun, Tianhang Xiang, Zhuangwei Zhuang, Liuren Yin, Mingkui Tan\n", + "Abstract: We study the task of extending the large language model (LLM) into a\n", + "vision-language instruction-following model. This task is crucial but\n", + "challenging since the LLM is trained on text modality only, making it hard to\n", + "effectively digest the visual modality. To address this, existing methods\n", + "typically train a visual adapter to align the representation between a\n", + "pre-trained vision transformer (ViT) and the LLM by a generative image\n", + "captioning loss. However, we find that the generative objective can only\n", + "produce weak alignment for vision and language, making the aligned\n", + "vision-language model very hungry for the instruction fine-tuning data. In this\n", + "paper, we propose CG-VLM that applies both Contrastive and Generative alignment\n", + "objectives to effectively align the representation of ViT and LLM. Different\n", + "from image level and sentence level alignment in common contrastive learning\n", + "settings, CG-VLM aligns the image-patch level features and text-token level\n", + "embeddings, which, however, is very hard to achieve as no explicit grounding\n", + "patch-token relation provided in standard image captioning datasets. To address\n", + "this issue, we propose to maximize the averaged similarity between pooled\n", + "image-patch features and text-token embeddings. Extensive experiments\n", + "demonstrate that the proposed CG-VLM produces strong vision-language alignment\n", + "and is an efficient instruction learner. For example, using only 10%\n", + "instruction tuning data, we reach 95% performance of state-of-the-art method\n", + "LLaVA [29] on the zero-shot ScienceQA-Image benchmark.\n", + "URL: http://arxiv.org/abs/2311.17945v1\n", + "Published: 2023-11-29 03:29:46+00:00\n", + "\n", + "Title: Mitigating Hallucination in Visual Language Models with Visual Supervision\n", + "Authors: Zhiyang Chen, Yousong Zhu, Yufei Zhan, Zhaowen Li, Chaoyang Zhao, Jinqiao Wang, Ming Tang\n", + "Abstract: Large vision-language models (LVLMs) suffer from hallucination a lot,\n", + "generating responses that apparently contradict to the image content\n", + "occasionally. The key problem lies in its weak ability to comprehend detailed\n", + "content in a multi-modal context, which can be mainly attributed to two factors\n", + "in training data and loss function. The vision instruction dataset primarily\n", + "focuses on global description, and the auto-regressive loss function favors\n", + "text modeling rather than image understanding. In this paper, we bring more\n", + "detailed vision annotations and more discriminative vision models to facilitate\n", + "the training of LVLMs, so that they can generate more precise responses without\n", + "encounter hallucination. On one hand, we generate image-text pairs with\n", + "detailed relationship annotations in panoptic scene graph dataset (PSG). These\n", + "conversations pay more attention on detailed facts in the image, encouraging\n", + "the model to answer questions based on multi-modal contexts. On the other hand,\n", + "we integrate SAM and mask prediction loss as auxiliary supervision, forcing the\n", + "LVLMs to have the capacity to identify context-related objects, so that they\n", + "can generate more accurate responses, mitigating hallucination. Moreover, to\n", + "provide a deeper evaluation on the hallucination in LVLMs, we propose a new\n", + "benchmark, RAH-Bench. It divides vision hallucination into three different\n", + "types that contradicts the image with wrong categories, attributes or\n", + "relations, and introduces False Positive Rate as detailed sub-metric for each\n", + "type. In this benchmark, our approach demonstrates an +8.4% enhancement\n", + "compared to original LLaVA and achieves widespread performance improvements\n", + "across other models.\n", + "URL: http://arxiv.org/abs/2311.16479v1\n", + "Published: 2023-11-27 09:30:02+00:00\n", "\n", "Title: PG-Video-LLaVA: Pixel Grounding Large Video-Language Models\n", "Authors: Shehan Munasinghe, Rusiru Thushara, Muhammad Maaz, Hanoona Abdul Rasheed, Salman Khan, Mubarak Shah, Fahad Khan\n", @@ -650,36 +663,189 @@ "advantages to the video domain, delivering promising gains on video-based\n", "conversation and grounding tasks. Project Page:\n", "https://github.com/mbzuai-oryx/Video-LLaVA\n", - "Published: 2023-11-22 14:48:30+00:00\n", "URL: http://arxiv.org/abs/2311.13435v1\n", + "Published: 2023-11-22 14:48:30+00:00\n", "\n", + "Title: ShareGPT4V: Improving Large Multi-Modal Models with Better Captions\n", + "Authors: Lin Chen, Jinsong Li, Xiaoyi Dong, Pan Zhang, Conghui He, Jiaqi Wang, Feng Zhao, Dahua Lin\n", + "Abstract: In the realm of large multi-modal models (LMMs), efficient modality alignment\n", + "is crucial yet often constrained by the scarcity of high-quality image-text\n", + "data. To address this bottleneck, we introduce the ShareGPT4V dataset, a\n", + "pioneering large-scale resource featuring 1.2 million highly descriptive\n", + "captions, which surpasses existing datasets in diversity and information\n", + "content, covering world knowledge, object properties, spatial relationships,\n", + "and aesthetic evaluations. Specifically, ShareGPT4V originates from a curated\n", + "100K high-quality captions collected from advanced GPT4-Vision and has been\n", + "expanded to 1.2M with a superb caption model trained on this subset. ShareGPT4V\n", + "first demonstrates its effectiveness for the Supervised Fine-Tuning (SFT)\n", + "phase, by substituting an equivalent quantity of detailed captions in existing\n", + "SFT datasets with a subset of our high-quality captions, significantly\n", + "enhancing the LMMs like LLaVA-7B, LLaVA-1.5-13B, and Qwen-VL-Chat-7B on the MME\n", + "and MMBench benchmarks, with respective gains of 222.8/22.0/22.3 and\n", + "2.7/1.3/1.5. We further incorporate ShareGPT4V data into both the pre-training\n", + "and SFT phases, obtaining ShareGPT4V-7B, a superior LMM based on a simple\n", + "architecture that has remarkable performance across a majority of the\n", + "multi-modal benchmarks. This project is available at\n", + "https://ShareGPT4V.github.io to serve as a pivotal resource for advancing the\n", + "LMMs community.\n", + "URL: http://arxiv.org/abs/2311.12793v2\n", + "Published: 2023-11-21 18:58:11+00:00\n", + "\n", + "Title: VLM-Eval: A General Evaluation on Video Large Language Models\n", + "Authors: Shuailin Li, Yuang Zhang, Yucheng Zhao, Qiuyue Wang, Fan Jia, Yingfei Liu, Tiancai Wang\n", + "Abstract: Despite the rapid development of video Large Language Models (LLMs), a\n", + "comprehensive evaluation is still absent. In this paper, we introduce a unified\n", + "evaluation that encompasses multiple video tasks, including captioning,\n", + "question and answering, retrieval, and action recognition. In addition to\n", + "conventional metrics, we showcase how GPT-based evaluation can match human-like\n", + "performance in assessing response quality across multiple aspects. We propose a\n", + "simple baseline: Video-LLaVA, which uses a single linear projection and\n", + "outperforms existing video LLMs. Finally, we evaluate video LLMs beyond\n", + "academic datasets, which show encouraging recognition and reasoning\n", + "capabilities in driving scenarios with only hundreds of video-instruction pairs\n", + "for fine-tuning. We hope our work can serve as a unified evaluation for video\n", + "LLMs, and help expand more practical scenarios. The evaluation code will be\n", + "available soon.\n", + "URL: http://arxiv.org/abs/2311.11865v1\n", + "Published: 2023-11-20 16:02:10+00:00\n", + "\n", + "Title: Video-LLaVA: Learning United Visual Representation by Alignment Before Projection\n", + "Authors: Bin Lin, Yang Ye, Bin Zhu, Jiaxi Cui, Munan Ning, Peng Jin, Li Yuan\n", + "Abstract: The Large Vision-Language Model (LVLM) has enhanced the performance of\n", + "various downstream tasks in visual-language understanding. Most existing\n", + "approaches encode images and videos into separate feature spaces, which are\n", + "then fed as inputs to large language models. However, due to the lack of\n", + "unified tokenization for images and videos, namely misalignment before\n", + "projection, it becomes challenging for a Large Language Model (LLM) to learn\n", + "multi-modal interactions from several poor projection layers. In this work, we\n", + "unify visual representation into the language feature space to advance the\n", + "foundational LLM towards a unified LVLM. As a result, we establish a simple but\n", + "robust LVLM baseline, Video-LLaVA, which learns from a mixed dataset of images\n", + "and videos, mutually enhancing each other. Video-LLaVA achieves superior\n", + "performances on a broad range of 9 image benchmarks across 5 image\n", + "question-answering datasets and 4 image benchmark toolkits. Additionally, our\n", + "Video-LLaVA also outperforms Video-ChatGPT by 5.8%, 9.9%, 18.6%, and 10.1% on\n", + "MSRVTT, MSVD, TGIF, and ActivityNet, respectively. Notably, extensive\n", + "experiments demonstrate that Video-LLaVA mutually benefits images and videos\n", + "within a unified visual representation, outperforming models designed\n", + "specifically for images or videos. We aim for this work to provide modest\n", + "insights into the multi-modal inputs for the LLM.\n", + "URL: http://arxiv.org/abs/2311.10122v2\n", + "Published: 2023-11-16 10:59:44+00:00\n", + "\n", + "Title: Unlock the Power: Competitive Distillation for Multi-Modal Large Language Models\n", + "Authors: Xinwei Li, Li Lin, Shuai Wang, Chen Qian\n", + "Abstract: Recently, multi-modal content generation has attracted lots of attention from\n", + "researchers by investigating the utilization of visual instruction tuning based\n", + "on large language models (LLMs). To enhance the performance and generalization\n", + "ability of such LLMs, the practice of distilling knowledge from pretrained\n", + "multi-modal models (a.k.a. teachers) to more compact multi-modal LLMs\n", + "(students) has gained considerable interest. However, the prevailing paradigm\n", + "of instructiontuning in multi-modal LLMs knowledge distillation is\n", + "resource-intensive and unidirectional, neglecting the potential for mutual\n", + "feedback between the student and teacher models. Thus, we propose an innovative\n", + "Competitive Multi-modal Distillation framework (CoMD), which captures\n", + "bidirectional feedback between teacher and student models and continually\n", + "updates the multi-modal capabilities that the student model has learned. It\n", + "comprises two stages: multi-modal pre-training and multi-modal competitive\n", + "distillation. The first stage pre-trains the student model on a large number of\n", + "filtered multi-modal datasets. The second stage facilitates a bidirectional\n", + "knowledge transfer between the student and teacher models. Our experimental\n", + "analysis of diverse datasets shows that our knowledge transfer method\n", + "consistently improves the capabilities of the student model. Finally, the\n", + "7B-sized student model after four distillations surpassed the current\n", + "state-of-the-art model LLaVA-13B on the ScienceQA and LLaVA Test dataset, also\n", + "outperforms other strong baselines in the zero-shot setting.\n", + "URL: http://arxiv.org/abs/2311.08213v1\n", + "Published: 2023-11-14 14:49:46+00:00\n", + "\n", + "Title: Improving hateful memes detection via learning hatefulness-aware embedding space through retrieval-guided contrastive learning\n", + "Authors: Jingbiao Mei, Jinghong Chen, Weizhe Lin, Bill Byrne, Marcus Tomalin\n", + "Abstract: Hateful memes have emerged as a significant concern on the Internet. These\n", + "memes, which are a combination of image and text, often convey messages vastly\n", + "different from their individual meanings. Thus, detecting hateful memes\n", + "requires the system to jointly understand the visual and textual modalities.\n", + "However, our investigation reveals that the embedding space of existing\n", + "CLIP-based systems lacks sensitivity to subtle differences in memes that are\n", + "vital for correct hatefulness classification. To address this issue, we propose\n", + "constructing a hatefulness-aware embedding space through retrieval-guided\n", + "contrastive training. Specifically, we add an auxiliary loss that utilizes hard\n", + "negative and pseudo-gold samples to train the embedding space. Our approach\n", + "achieves state-of-the-art performance on the HatefulMemes dataset with an AUROC\n", + "of 86.7. Notably, our approach outperforms much larger fine-tuned Large\n", + "Multimodal Models like Flamingo and LLaVA. Finally, we demonstrate a\n", + "retrieval-based hateful memes detection system, which is capable of making\n", + "hatefulness classification based on data unseen in training from a database.\n", + "This allows developers to update the hateful memes detection system by simply\n", + "adding new data without retraining, a desirable feature for real services in\n", + "the constantly-evolving landscape of hateful memes on the Internet.\n", + "URL: http://arxiv.org/abs/2311.08110v1\n", + "Published: 2023-11-14 12:14:54+00:00\n", + "\n", + "Title: To See is to Believe: Prompting GPT-4V for Better Visual Instruction Tuning\n", + "Authors: Junke Wang, Lingchen Meng, Zejia Weng, Bo He, Zuxuan Wu, Yu-Gang Jiang\n", + "Abstract: Existing visual instruction tuning methods typically prompt large language\n", + "models with textual descriptions to generate instruction-following data.\n", + "Despite the promising performance achieved, these descriptions are derived from\n", + "image annotations, which are oftentimes coarse-grained. Furthermore, the\n", + "instructions might even contradict the visual content without observing the\n", + "entire visual context. To address this challenge, we introduce a fine-grained\n", + "visual instruction dataset, LVIS-Instruct4V, which contains 220K visually\n", + "aligned and context-aware instructions produced by prompting the powerful\n", + "GPT-4V with images from LVIS. Through experimental validation and case studies,\n", + "we demonstrate that high-quality visual instructional data could improve the\n", + "performance of LLaVA-1.5, a state-of-the-art large multimodal model, across a\n", + "wide spectrum of benchmarks by clear margins. Notably, by simply replacing the\n", + "LLaVA-Instruct with our LVIS-Instruct4V, we achieve better results than LLaVA\n", + "on most challenging LMM benchmarks, e.g., LLaVA$^w$ (76.7 vs. 70.7) and MM-Vet\n", + "(40.2 vs. 35.4). We release our data and model at\n", + "https://github.com/X2FD/LVIS-INSTRUCT4V.\n", + "URL: http://arxiv.org/abs/2311.07574v2\n", + "Published: 2023-11-13 18:59:31+00:00\n", + "\n", + "Title: MEGAVERSE: Benchmarking Large Language Models Across Languages, Modalities, Models and Tasks\n", + "Authors: Sanchit Ahuja, Divyanshu Aggarwal, Varun Gumma, Ishaan Watts, Ashutosh Sathe, Millicent Ochieng, Rishav Hada, Prachi Jain, Maxamed Axmed, Kalika Bali, Sunayana Sitaram\n", + "Abstract: Recently, there has been a rapid advancement in research on Large Language\n", + "Models (LLMs), resulting in significant progress in several Natural Language\n", + "Processing (NLP) tasks. Consequently, there has been a surge in LLM evaluation\n", + "research to comprehend the models' capabilities and limitations. However, much\n", + "of this research has been confined to the English language, leaving LLM\n", + "building and evaluation for non-English languages relatively unexplored. There\n", + "has been an introduction of several new LLMs, necessitating their evaluation on\n", + "non-English languages. This study aims to expand our MEGA benchmarking suite by\n", + "including six new datasets to form the MEGAVERSE benchmark. The benchmark\n", + "comprises 22 datasets covering 81 languages, including low-resource African\n", + "languages. We evaluate several state-of-the-art LLMs like GPT-3.5-Turbo, GPT4,\n", + "PaLM2, and Llama2 on the MEGAVERSE datasets. Additionally, we include two\n", + "multimodal datasets in the benchmark and assess the performance of the\n", + "LLaVa-v1.5 model. Our experiments suggest that GPT4 and PaLM2 outperform the\n", + "Llama models on various tasks, notably on low-resource languages, with GPT4\n", + "outperforming PaLM2 on more datasets than vice versa. However, issues such as\n", + "data contamination must be addressed to obtain an accurate assessment of LLM\n", + "performance on non-English languages.\n", + "URL: http://arxiv.org/abs/2311.07463v1\n", + "Published: 2023-11-13 16:45:37+00:00\n", "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[33mDomain_expert\u001b[0m (to chat_manager):\n", - "\n", - "Based on the information provided, the paper titled \"PG-Video-LLaVA: Pixel Grounding Large Video-Language Models\" presents a Large Multimodal Model (LMM) that extends image-based LMMs to videos by incorporating pixel-level grounding capabilities and integrating audio cues through text transcription to enhance video-context understanding.\n", - "\n", - "The potential applications of Video-LLaVA in multi-modal models are significant:\n", - "\n", - "1. **Enhanced Video Understanding**: Video-LLaVA's ability to spatially and temporally localize objects in videos can be applied to improve video analysis systems, such as those used in surveillance, sports analytics, and content moderation.\n", - "\n", - "2. **Interactive Video Systems**: The grounding module allows for interactive systems where users can query specific objects or events within a video, which could be useful in educational platforms, interactive storytelling, or customer service bots that interact with video content.\n", "\n", - "3. **Accessibility**: By transcribing audio signals and grounding visual content, Video-LLaVA can aid in creating more accessible content for individuals with hearing or visual impairments, providing richer descriptions of video content.\n", "\n", - "4. **Content Creation and Editing**: The model's grounding capabilities can be leveraged in video editing software to allow creators to manipulate video content through natural language commands, streamlining the editing process.\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mData_scientist\u001B[0m (to chat_manager):\n", "\n", - "5. **Video-Based Conversational Agents**: The paper mentions video-based conversation benchmarking, which implies that Video-LLaVA could be used to develop conversational agents that understand and respond to video content, enhancing customer support or entertainment applications.\n", + "Based on the search results, it appears that \"LLaVA\" is a term related to Large Language Models (LLMs) and their applications in vision-language tasks. The papers listed discuss various aspects of LLaVA and its applications, including instruction learning, hallucination mitigation, video understanding, and more.\n", "\n", - "6. **Research and Education**: In academic settings, Video-LLaVA could be used to analyze educational videos, providing summaries, answering questions, and helping students locate information within video lectures.\n", + "From the abstracts, we can see that LLaVA and its variants are being used to improve the alignment between visual and language representations, which is crucial for tasks such as image captioning, visual question answering, and video understanding. These models are designed to process and understand multi-modal data, combining visual information with textual instructions or queries.\n", "\n", - "7. **Marketing and Advertising**: Companies could use Video-LLaVA to analyze consumer videos for product placement, sentiment analysis, and to understand consumer behavior through video content.\n", + "For example, the paper titled \"Contrastive Vision-Language Alignment Makes Efficient Instruction Learner\" discusses how to align the representation of a Vision Transformer (ViT) with an LLM to create an efficient instruction learner for vision-language tasks. Another paper, \"PG-Video-LLaVA: Pixel Grounding Large Video-Language Models,\" extends the capabilities of LLaVA to videos, enabling the model to spatially and temporally localize objects in videos following user instructions.\n", "\n", - "8. **Integration with Robotics**: Robots equipped with video processing capabilities could use Video-LLaVA to better understand their environment and interact with it based on visual and audio cues.\n", + "The potential applications in computer vision are vast and include:\n", "\n", - "The paper's introduction of new benchmarks for prompt-based object grounding performance in videos also suggests that Video-LLaVA could push the boundaries of what is possible in video understanding and interaction, leading to more sophisticated multi-modal applications.\n", + "1. Image and video captioning: Generating descriptive text for images and videos.\n", + "2. Visual question answering: Answering questions based on visual content.\n", + "3. Object detection and localization: Identifying and locating objects in images and videos.\n", + "4. Video understanding: Interpreting actions, events, and narratives in video content.\n", + "5. Hallucination mitigation: Reducing instances where the model generates responses that contradict the visual content.\n", "\n", - "For further analysis, one would need to delve into the technical details of the paper, experiment with the model, and possibly contribute to the project, which is available on the provided GitHub page.\n", + "These applications are crucial for developing more intelligent and interactive AI systems that can understand and respond to visual content in a human-like manner. The research on LLaVA and related models is contributing to the advancement of multi-modal AI, which can have significant implications for fields such as autonomous vehicles, assistive technologies, content moderation, and more.\n", "\n", "TERMINATE\n", "\n", @@ -689,8 +855,13 @@ } ], "source": [ - "new_builder = AgentBuilder(config_path=config_path).load(saved_path)\n", - "new_builder.start(task=\"Find a latest paper about Llava on arxiv and find its potential applications in multi-modal model.\")\n", + "new_builder = AgentBuilder(config_path=config_path)\n", + "agent_list, agent_configs = new_builder.load(saved_path) # load previous agent configs\n", + "start_task(\n", + " execution_task=\"Find a recent paper about Llava on arxiv and find its potential applications in computer vision.\",\n", + " agent_list=agent_list,\n", + " llm_config=default_llm_config\n", + ")\n", "new_builder.clear_all_agents()" ] }, @@ -709,14 +880,14 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 15, "id": "4051c25b2cd1918c", "metadata": { + "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-28T09:26:01.069144900Z", - "start_time": "2023-11-28T09:23:39.355869200Z" - }, - "collapsed": false + "end_time": "2023-12-03T13:59:35.497212500Z", + "start_time": "2023-12-03T13:47:45.765859300Z" + } }, "outputs": [ { @@ -724,10 +895,10 @@ "output_type": "stream", "text": [ "Generating agents...\n", - "Data_scientist,Domain_expert,Software_engineer are generated.\n", + "Data_scientist,Machine_learning_engineer,Research_analyst are generated.\n", "Preparing configuration for Data_scientist...\n", - "Preparing configuration for Domain_expert...\n", - "Preparing configuration for Software_engineer...\n", + "Preparing configuration for Machine_learning_engineer...\n", + "Preparing configuration for Research_analyst...\n", "Creating agent Data_scientist with backbone gpt-4-1106-preview...\n" ] }, @@ -742,186 +913,187 @@ "name": "stdout", "output_type": "stream", "text": [ - "Creating agent Domain_expert with backbone gpt-4-1106-preview...\n", - "Creating agent Software_engineer with backbone gpt-4-1106-preview...\n", - "Adding user console proxy...\n", - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", - "Find a latest paper about XAI on arxiv and find its potential applications in medical.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", - "\n", - "To accomplish the task of finding the latest paper about Explainable Artificial Intelligence (XAI) on arXiv and evaluating its potential applications in the medical field, we will break down the task into several steps:\n", - "\n", - "1. **Search arXiv for the latest papers on XAI**: Use the arXiv API to search for papers that match the topic \"Explainable Artificial Intelligence\".\n", - "2. **Download and read the paper's summary**: Extract and present the content of the paper's summary to gain insight into its subject matter.\n", - "3. **Evaluate real-world applications in medicine**: Analyze the content for potential applications within the medical field.\n", - "\n", - "Let's start with the first step:\n", - "\n", - "### Step 1 - Searching for the latest XAI papers on arXiv\n", - "\n", - "We will use the `arxiv` Python library to search for papers. If you don't have it installed, you can install it using `pip install arxiv`.\n", - "\n", - "```python\n", - "import arxiv\n", - "import datetime\n", - "\n", - "# Define the query parameters\n", - "query = \"cat:cs.AI AND ti:XAI\"\n", - "sort_by = arxiv.SortCriterion.SubmittedDate # Sort by most recent submissions\n", - "\n", - "# Search for papers on arXiv that match the query\n", - "search = arxiv.Search(\n", - " query=query,\n", - " sort_by=sort_by,\n", - " sort_order=arxiv.SortOrder.Descending,\n", - " max_results=1\n", - ")\n", - "\n", - "# Fetch the latest paper\n", - "for paper in search.results():\n", - " print(f\"Title: {paper.title}\")\n", - " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", - " print(f\"Submitted Date: {paper.published}\")\n", - " print(f\"Summary: {paper.summary.replace('\\n', ' ')}\")\n", - " print(f\"URL: {paper.entry_id}\")\n", - " break # We only want the latest paper\n", - "```\n", - "\n", - "Once you run this script, it should output the title, authors, submission date, and summary of the latest paper about Explainable Artificial Intelligence on arXiv, as well as provide a URL to the paper.\n", - "\n", - "### Step 2 - Reading the paper's summary\n", - "\n", - "The script above already takes care of downloading and presenting the summary of the latest paper. This will provide us with an overview of the paper's content and potential applications.\n", - "\n", - "### Step 3 - Evaluating potential medical applications\n", - "\n", - "For evaluating real-world applications in medicine, we would need to read and understand the content of the paper's summary. Especially, we would look for:\n", - "- Any mention of medical datasets or case studies within the paper summary.\n", - "- Generalizable findings or methods that could be applied to medical scenarios, such as diagnostic tools, drug discovery, patient data analysis, etc.\n", - "\n", - "After you have executed the Python script and obtained the latest paper's summary information, review the content to determine if there are potential applications in the medical field. I can guide you through this analysis by asking you to consider the following questions:\n", - "\n", - "- Does the paper discuss applications or case studies related to medicine or healthcare?\n", - "- Are the methods or approaches described in the paper suitable for tasks in the medical domain, such as interpreting medical images, analyzing genomic data, or personalizing treatment plans?\n", - "\n", - "If you share the summary with me, I can help in conceptualizing how the findings in the paper might be applied to the medical field. However, I cannot analyse the content myself, you will have to apply this analysis based on my guidance.\n", - "\n", - "Please run the code provided to find the latest paper on XAI from arXiv and share the summary with me for further guidance on potential medical applications.\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" + "Creating agent Machine_learning_engineer with backbone gpt-4-1106-preview...\n", + "Creating agent Research_analyst with backbone gpt-4-1106-preview...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" + "Multiple assistants with name Research_analyst found. Using the first assistant in the list. Please specify the assistant ID in llm_config to use a specific assistant.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "Adding user console proxy...\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", + "Find a recent paper about XAI on arxiv and find its potential applications in medical.\n", "\n", - "exitcode: 1 (execution failed)\n", - "Code output: \n", - " File \"\", line 21\n", - " print(f\"Summary: {paper.summary.replace('\\n', ' ')}\")\n", - " ^\n", - "SyntaxError: f-string expression part cannot include a backslash\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mData_scientist\u001B[0m (to chat_manager):\n", + "\n", + "We will start by searching for a recent paper about Explainable Artificial Intelligence (XAI) on arXiv. To do this, I'll write a Python script that uses the arXiv API to fetch relevant papers. We will then look at the most recent paper and analyze its content to identify potential applications within the medical domain. \n", + "\n", + "Let's start by writing the script to search for papers on arXiv. This code will fetch papers related to XAI that are latest:\n", + "\n", + "```python\n", + "import urllib\n", + "import feedparser\n", + "\n", + "# Base api query url\n", + "base_url = 'http://export.arxiv.org/api/query?'\n", + "\n", + "# Search parameters\n", + "search_query = 'all:XAI AND cat:cs.AI' # search for XAI in the field of computer science artificial intelligence\n", + "start = 0 # start at the first result\n", + "total_results = 1 # want only the most recent result\n", + "results_per_iteration = 1 # results per query fetch\n", + "wait_time = 5 # seconds to wait between fetches, as a courtesy to arXiv's servers\n", + "\n", + "# Form the full query url\n", + "query = f'search_query={search_query}&start={start}&max_results={total_results}'\n", + "\n", + "# Perform a GET request to the arXiv API and parse the results\n", + "response = urllib.request.urlopen(base_url+query).read()\n", + "feed = feedparser.parse(response)\n", + "\n", + "# Print out the information from the most recent paper on XAI\n", + "for entry in feed.entries:\n", + " print('Title: %s' % entry.title)\n", + " print('Authors: %s' % ', '.join(author.name for author in entry.authors))\n", + " print('Published: %s' % entry.published)\n", + " print('Summary: %s' % entry.summary)\n", + " print('arXiv ID: %s' % entry.id.split('/abs/')[-1])\n", + " print('Link: %s' % entry.id)\n", + "```\n", "\n", + "Please run this script in a Python environment to fetch the information about the most recent paper on XAI. After running this code, we will get the title, authors, publication date, summary, arXiv ID, and a link to the paper. Once we have the paper information, we can proceed to analyze it and discuss potential medical applications.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mSoftware_engineer\u001b[0m (to chat_manager):\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", - "It appears there was an error due to the use of a backslash within an f-string expression. Let's correct the script to properly print the summary without causing a syntax error. We can use double quotes for the string inside the `replace` method to avoid this issue.\n", + "exitcode: 1 (execution failed)\n", + "Code output: \n", + "Traceback (most recent call last):\n", + " File \"\", line 18, in \n", + " response = urllib.request.urlopen(base_url+query).read()\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/urllib/request.py\", line 216, in urlopen\n", + " return opener.open(url, data, timeout)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/urllib/request.py\", line 519, in open\n", + " response = self._open(req, data)\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/urllib/request.py\", line 536, in _open\n", + " result = self._call_chain(self.handle_open, protocol, protocol +\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/urllib/request.py\", line 496, in _call_chain\n", + " result = func(*args)\n", + " ^^^^^^^^^^^\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/urllib/request.py\", line 1377, in http_open\n", + " return self.do_open(http.client.HTTPConnection, req)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/urllib/request.py\", line 1348, in do_open\n", + " h.request(req.get_method(), req.selector, req.data, headers,\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/http/client.py\", line 1286, in request\n", + " self._send_request(method, url, body, headers, encode_chunked)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/http/client.py\", line 1297, in _send_request\n", + " self.putrequest(method, url, **skips)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/http/client.py\", line 1131, in putrequest\n", + " self._validate_path(url)\n", + " File \"/home/elpis_ubuntu/anaconda3/envs/llm/lib/python3.11/http/client.py\", line 1231, in _validate_path\n", + " raise InvalidURL(f\"URL can't contain control characters. {url!r} \"\n", + "http.client.InvalidURL: URL can't contain control characters. '/api/query?search_query=all:XAI AND cat:cs.AI&start=0&max_results=1' (found at least ' ')\n", "\n", - "Here is the corrected Python script:\n", "\n", - "```python\n", - "import arxiv\n", - "import datetime\n", + "--------------------------------------------------------------------------------\n", + "\u001B[33mData_scientist\u001B[0m (to chat_manager):\n", "\n", - "# Define the query parameters\n", - "query = \"cat:cs.AI AND ti:XAI\"\n", - "sort_by = arxiv.SortCriterion.SubmittedDate # Sort by most recent submissions\n", + "It seems there's an issue with the URL encoding in the script when making the request to the arXiv API. The query parameters need to be properly encoded to ensure that spaces and special characters are handled correctly. Let's correct the script by encoding the query parameters.\n", "\n", - "# Search for papers on arXiv that match the query\n", - "search = arxiv.Search(\n", - " query=query,\n", - " sort_by=sort_by,\n", - " sort_order=arxiv.SortOrder.Descending,\n", - " max_results=1\n", - ")\n", + "Below is the corrected Python script. Please run it to fetch the information about the most recent paper on Explainable Artificial Intelligence (XAI) from arXiv:\n", "\n", - "# Fetch the latest paper\n", - "for paper in search.results():\n", - " print(f\"Title: {paper.title}\")\n", - " print(f\"Authors: {', '.join(author.name for author in paper.authors)}\")\n", - " print(f\"Submitted Date: {paper.published}\")\n", - " summary = paper.summary.replace(\"\\n\", \" \")\n", - " print(f\"Summary: {summary}\")\n", - " print(f\"URL: {paper.entry_id}\")\n", - " break # We only want the latest paper\n", + "```python\n", + "import urllib.parse\n", + "import urllib.request\n", + "import feedparser\n", + "\n", + "# Base api query url\n", + "base_url = 'http://export.arxiv.org/api/query?'\n", + "\n", + "# Search parameters\n", + "search_query = 'all:XAI AND cat:cs.AI' # search for XAI in the field of computer science artificial intelligence\n", + "start = 0 # start at the first result\n", + "total_results = 1 # want only the most recent result\n", + "results_per_iteration = 1 # results per query fetch\n", + "\n", + "# Encode the query parameters\n", + "query = f'search_query={urllib.parse.quote(search_query)}&start={start}&max_results={total_results}'\n", + "\n", + "# Perform a GET request to the arXiv API and parse the results\n", + "response = urllib.request.urlopen(base_url+query).read()\n", + "feed = feedparser.parse(response)\n", + "\n", + "# Print out the information from the most recent paper on XAI\n", + "for entry in feed.entries:\n", + " print('Title:', entry.title)\n", + " print('Authors:', ', '.join(author.name for author in entry.authors))\n", + " print('Published:', entry.published)\n", + " print('Summary:', entry.summary)\n", + " print('arXiv ID:', entry.id.split('/abs/')[-1])\n", + " print('Link:', entry.id)\n", "```\n", "\n", - "Please run the corrected script to get the details of the latest paper regarding Explainable Artificial Intelligence (XAI) on arXiv. Once you obtain the summary, we can discuss how to conceptualize its real-world applications in the medical field.\n", + "This updated script ensures that the search query is properly encoded before making the request to arXiv. Once you run the updated version, you should be able to retrieve the details of the most recent paper on XAI. We can then move to the analysis part and discuss potential applications in the medical field.\n", + "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mUser_console_and_Python_code_interpreter\u001b[0m (to chat_manager):\n", + "\u001B[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001B[0m\n", + "\u001B[33mUser_console_and_Python_code_interpreter\u001B[0m (to chat_manager):\n", "\n", "exitcode: 0 (execution succeeded)\n", "Code output: \n", - "Title: Forms of Understanding of XAI-Explanations\n", - "Authors: Hendrik Buschmeier, Heike M. Buhl, Friederike Kern, Angela Grimminger, Helen Beierling, Josephine Fisher, André Groß, Ilona Horwath, Nils Klowait, Stefan Lazarov, Michael Lenke, Vivien Lohmer, Katharina Rohlfing, Ingrid Scharlau, Amit Singh, Lutz Terfloth, Anna-Lisa Vollmer, Yu Wang, Annedore Wilmes, Britta Wrede\n", - "Submitted Date: 2023-11-15 08:06:51+00:00\n", - "Summary: Explainability has become an important topic in computer science and artificial intelligence, leading to a subfield called Explainable Artificial Intelligence (XAI). The goal of providing or seeking explanations is to achieve (better) 'understanding' on the part of the explainee. However, what it means to 'understand' is still not clearly defined, and the concept itself is rarely the subject of scientific investigation. This conceptual article aims to present a model of forms of understanding in the context of XAI and beyond. From an interdisciplinary perspective bringing together computer science, linguistics, sociology, and psychology, a definition of understanding and its forms, assessment, and dynamics during the process of giving everyday explanations are explored. Two types of understanding are considered as possible outcomes of explanations, namely enabledness, 'knowing how' to do or decide something, and comprehension, 'knowing that' -- both in different degrees (from shallow to deep). Explanations regularly start with shallow understanding in a specific domain and can lead to deep comprehension and enabledness of the explanandum, which we see as a prerequisite for human users to gain agency. In this process, the increase of comprehension and enabledness are highly interdependent. Against the background of this systematization, special challenges of understanding in XAI are discussed.\n", - "URL: http://arxiv.org/abs/2311.08760v1\n", + "Title: A Critical Survey on Fairness Benefits of XAI\n", + "Authors: Luca Deck, Jakob Schoeffer, Maria De-Arteaga, Niklas Kühl\n", + "Published: 2023-10-15T08:17:45Z\n", + "Summary: In this critical survey, we analyze typical claims on the relationship\n", + "between explainable AI (XAI) and fairness to disentangle the multidimensional\n", + "relationship between these two concepts. Based on a systematic literature\n", + "review and a subsequent qualitative content analysis, we identify seven\n", + "archetypal claims from 175 papers on the alleged fairness benefits of XAI. We\n", + "present crucial caveats with respect to these claims and provide an entry point\n", + "for future discussions around the potentials and limitations of XAI for\n", + "specific fairness desiderata. While the literature often suggests XAI to be an\n", + "enabler for several fairness desiderata, we notice a divide between these\n", + "desiderata and the capabilities of XAI. We encourage to conceive XAI as one of\n", + "many tools to approach the multidimensional, sociotechnical challenge of\n", + "algorithmic fairness and to be more specific about how exactly what kind of XAI\n", + "method enables whom to address which fairness desideratum.\n", + "arXiv ID: 2310.13007v4\n", + "Link: http://arxiv.org/abs/2310.13007v4\n", "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001b[33mDomain_expert\u001b[0m (to chat_manager):\n", + "\u001B[33mResearch_analyst\u001B[0m (to chat_manager):\n", "\n", - "Based on the summary of the paper titled \"Forms of Understanding of XAI-Explanations,\" we can glean that this paper is conceptual in nature, aiming to define and explore the concept of 'understanding' in the context of Explainable Artificial Intelligence (XAI). The interdisciplinary perspective adopted by the authors examines the dynamics of understanding during the process of providing explanations, particularly focusing on two types of understanding: enabledness (knowing how) and comprehension (knowing that), ranging from shallow to deep levels.\n", + "Based on the output provided, the most recent paper on Explainable Artificial Intelligence (XAI) from arXiv is titled \"A Critical Survey on Fairness Benefits of XAI.\" It was authored by Luca Deck, Jakob Schoeffer, Maria De-Arteaga, and Niklas Kühl, and published on October 15, 2023.\n", "\n", - "While the paper does not explicitly target the medical field, we can conceptualize its potential applications in medicine by considering how these concepts could enhance AI's role in healthcare:\n", + "The summary discusses a critical survey conducted to analyze claims about the relationship between XAI and fairness. Through a systematic literature review and qualitative content analysis, the authors identified seven archetypal claims from 175 papers about the supposed fairness benefits of XAI. They present significant limitations and caveats regarding these claims, challenging the notion that XAI is a straightforward solution for fairness issues. The paper suggests reconsidering the role of XAI as one of the many tools to address the complex, sociotechnical challenge of algorithmic fairness. It emphasizes the importance of being specific about how certain XAI methods enable stakeholders to address particular fairness desiderata.\n", "\n", - "1. **Patient Education**: Utilizing the concept of 'understanding' to explain AI-driven medical diagnoses or treatment plans to patients, enabling them to comprehend their condition and the suggested medical interventions. By converting complex medical data into understandable explanations, patients can make more informed decisions about their health.\n", + "Regarding potential applications in the medical field, one can infer from the summary that while the paper itself may not be directly focused on medical applications, its insights could be relevant. In healthcare, fairness is a critical concern due to the potential impact of biased algorithms on patient outcomes. XAI could help medical professionals and policymakers understand how AI models make predictions, which can be essential for identifying and mitigating biases in high-stakes decisions such as diagnosis, treatment planning, or resource allocation.\n", "\n", - "2. **Clinical Decision Support**: Improvement in clinician's adoption of AI tools for decision-making by employing explanations that focus on enabledness (knowing how) to use AI recommendations and comprehension (knowing that) to understand the underlying reasons for AI's advice.\n", + "While the summary does not provide explicit applications of XAI in medicine, understanding the interplay between AI explainability and fairness is undoubtedly beneficial in the context of ethical AI deployment in healthcare. Increased transparency through XAI could lead to more equitable healthcare algorithms, but this requires careful consideration of how the explainability ties into fairness outcomes, as indicated by the authors.\n", "\n", - "3. **Medical Study and Research**: Enabling medical researchers to leverage AI tools that can explain their outputs in a way that provides deep understanding, thereby increasing trust in AI's ability to identify patterns and offer insights in complex datasets like genomic information or epidemiological studies.\n", - "\n", - "4. **Training Healthcare Professionals**: Enhancing the educational tools for healthcare professionals by incorporating XAI systems that provide varying degrees of understanding, from basic conceptual learning to advanced clinical applications.\n", - "\n", - "5. **Ethics and Legal Compliance**: The ability to provide understandable explanations for AI's actions could facilitate compliance with ethical guidelines and legal requirements, especially in sensitive areas such as patient data handling or algorithmic decisions affecting patient care.\n", - "\n", - "In summary, while the paper presents a conceptual model of understanding applicable to XAI systems, its principles could potentially be adapted and used to develop AI tools that are more transparent, trusted, and user-friendly in the medical domain. Applications could span patient interaction, clinical support, medical research, educational initiatives, and compliance requirements, all of which benefit from a higher degree of explainability and understanding provided by AI systems.\n", + "For further analysis, I would recommend reading the full paper to extract detailed discussions of these issues, which might highlight more specific applications or considerations for the medical field.\n", "\n", "TERMINATE\n", "\n", - "\n", "--------------------------------------------------------------------------------\n", "All agents have been cleared.\n" ] @@ -929,20 +1101,24 @@ ], "source": [ "new_builder = AgentBuilder(config_path=config_path)\n", - "new_builder.build(building_task, default_llm_config, use_gpts=True) # Transfer to GPTs API.\n", - "new_builder.start(task=\"Find a latest paper about XAI on arxiv and find its potential applications in medical.\")\n", + "agent_list, agent_configs = new_builder.build(building_task, default_llm_config, use_gpts=True) # Transfer to GPTs API.\n", + "start_task(\n", + " execution_task=\"Find a recent paper about XAI on arxiv and find its potential applications in medical.\",\n", + " agent_list=agent_list,\n", + " llm_config=default_llm_config\n", + ")\n", "new_builder.clear_all_agents()" ] }, { "cell_type": "code", "execution_count": null, - "id": "57194293da4ae4e1", + "outputs": [], + "source": [], "metadata": { "collapsed": false }, - "outputs": [], - "source": [] + "id": "8fbfef9268fc5191" } ], "metadata": { diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index 3e6aba8a73f..a1833b5f18b 100644 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -32,8 +32,8 @@ def test_build(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( - "Find a paper on arxiv by programming, and analysis its application in some domain. " - "For example, find a latest paper about gpt-4 on arxiv " + "Find a paper on arxiv by programming, and analyze its application in some domain. " + "For example, find a recent paper about gpt-4 on arxiv " "and find its potential applications in software." ) builder.build( @@ -58,8 +58,8 @@ def test_build(): def test_save(): builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") building_task = ( - "Find a paper on arxiv by programming, and analysis its application in some domain. " - "For example, find a latest paper about gpt-4 on arxiv " + "Find a paper on arxiv by programming, and analyze its application in some domain. " + "For example, find a recent paper about gpt-4 on arxiv " "and find its potential applications in software." ) @@ -79,7 +79,6 @@ def test_save(): # check config format assert saved_configs.get("building_task", None) is not None assert saved_configs.get("agent_configs", None) is not None - assert saved_configs.get("manager_system_message", None) is not None assert saved_configs.get("coding", None) is not None assert saved_configs.get("default_llm_config", None) is not None @@ -97,15 +96,15 @@ def test_load(): e["name"]: {"model": e["model"], "system_message": e["system_message"]} for e in configs["agent_configs"] } - builder.load( + agent_list, loaded_agent_configs = builder.load( config_save_path, user_proxy_work_dir=f"{here}/test_agent_scripts", docker="python:3", ) # check config loading - assert builder.coding == configs["coding"] - for agent in builder.agent_procs_assign.values(): + assert loaded_agent_configs["coding"] == configs["coding"] + for agent in agent_list: agent_name = agent[0].name assert agent_configs.get(agent_name, None) is not None assert agent_configs[agent_name]["model"] == agent[0].llm_config["model"] @@ -129,24 +128,3 @@ def test_clear_agent(): # check if the agent cleared assert len(builder.agent_procs_assign) == 0 - - -@pytest.mark.skipif( - not OPENAI_INSTALLED, - reason="do not run when dependency is not installed", -) -def test_start(): - builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4") - config_save_path = f"{here}/example_test_agent_builder_config.json" - builder.load(config_save_path) - test_task = "Find a latest paper about gpt-4 on arxiv and find its potential applications in software." - - group_chat, _ = builder.start(task=test_task) - history = group_chat.messages.copy() - - assert history[0]["content"] == test_task - history.reverse() - for msg in history: - if msg["content"] != "": - assert "TERMINATE" in msg["content"] - break diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index 29fddb61852..5b9c9804b21 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -140,7 +140,7 @@ and specify it when initializing AgentBuilder. AgentBuilder will automatically set up an endpoint server for open-source LLM. Make sure you have sufficient GPUs resources. ## Use GPTs -[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task. +[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapt to new task or improve on the current task. AutoBuild also support GPTs api by adding `use_gpts=True` to the `build()` function. ```python # Transfer to GPTs API. From dda676045b9ed6c90125d198156aec62987b7feb Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sun, 3 Dec 2023 23:28:58 +0800 Subject: [PATCH 68/74] update test example --- test/agentchat/contrib/example_test_agent_builder_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/agentchat/contrib/example_test_agent_builder_config.json b/test/agentchat/contrib/example_test_agent_builder_config.json index 38e5e903122..109ea0f20be 100644 --- a/test/agentchat/contrib/example_test_agent_builder_config.json +++ b/test/agentchat/contrib/example_test_agent_builder_config.json @@ -1,5 +1,5 @@ { - "building_task": "Find a paper on arxiv by programming, and analysis its application in some domain. For example, find a latest paper about gpt-4 on arxiv and find its potential applications in software.", + "building_task": "Find a paper on arxiv by programming, and analyze its application in some domain. For example, find a recent paper about gpt-4 on arxiv and find its potential applications in software.", "agent_configs": [ { "name": "Data_scientist", From e00a38dabcf02ce9ad2d628cfb7fbb68ec4eb8cc Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Sun, 3 Dec 2023 23:53:05 +0800 Subject: [PATCH 69/74] update test --- test/agentchat/contrib/test_agent_builder.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index a1833b5f18b..3f0ee7cbbfc 100644 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -4,12 +4,13 @@ import sys from packaging.requirements import Requirement from autogen.agentchat.contrib.agent_builder import AgentBuilder +from autogen import UserProxyAgent sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402 here = os.path.abspath(os.path.dirname(__file__)) -oai_config_path = os.path.join(KEY_LOC, OAI_CONFIG_LIST) +oai_config_path = OAI_CONFIG_LIST # openai>=1 required try: @@ -104,11 +105,14 @@ def test_load(): # check config loading assert loaded_agent_configs["coding"] == configs["coding"] + if loaded_agent_configs["coding"] is True: + assert isinstance(agent_list[0], UserProxyAgent) + agent_list = agent_list[1:] for agent in agent_list: - agent_name = agent[0].name + agent_name = agent.name assert agent_configs.get(agent_name, None) is not None - assert agent_configs[agent_name]["model"] == agent[0].llm_config["model"] - assert agent_configs[agent_name]["system_message"] == agent[0].system_message + assert agent_configs[agent_name]["model"] == agent.llm_config["model"] + assert agent_configs[agent_name]["system_message"] == agent.system_message @pytest.mark.skipif( From 3231fb06b803ab4842025c80569610bcd1f97f51 Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Sun, 3 Dec 2023 15:54:17 -0800 Subject: [PATCH 70/74] Update agent_chat.md --- website/docs/Use-Cases/agent_chat.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/website/docs/Use-Cases/agent_chat.md b/website/docs/Use-Cases/agent_chat.md index 4cdf2f0a640..46b9f5bad35 100644 --- a/website/docs/Use-Cases/agent_chat.md +++ b/website/docs/Use-Cases/agent_chat.md @@ -83,9 +83,6 @@ The figure below shows six examples of applications built using AutoGen. Find a list of examples in this page: [Automated Agent Chat Examples](../Examples.md#automated-multi-agent-chat) -8. **Automatic Agent Building** - - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_autobuild.ipynb) - ## For Further Reading _Interested in the research that leads to this package? Please check the following papers._ From 9774235805ef9f8e36f0a2025501562efac48ced Mon Sep 17 00:00:00 2001 From: Jieyu Zhang Date: Sun, 3 Dec 2023 16:11:02 -0800 Subject: [PATCH 71/74] Update Examples.md --- website/docs/Examples.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 3bc13b7847c..0a0324303a4 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -70,7 +70,6 @@ Links to notebook examples: - Utility Functions to Help Managing API configurations effectively - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) - Cost Calculation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_client_cost.ipynb) - ### Inference Hyperparameters Tuning AutoGen offers a cost-effective hyperparameter optimization technique [EcoOptiGen](https://arxiv.org/abs/2303.04673) for tuning Large Language Models. The research study finds that tuning hyperparameters can significantly improve the utility of them. From 38cc1ff54430983d68a86612cbb29c211ec67b4d Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Mon, 4 Dec 2023 10:03:41 +0800 Subject: [PATCH 72/74] address issues from qingyun; update agent_builder.py; update notebook --- autogen/agentchat/contrib/agent_builder.py | 38 ++++++++++++---------- notebook/agentchat_autobuild.ipynb | 11 +++---- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index 7e53bde32bd..55590771a7d 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -11,7 +11,7 @@ class AgentBuilder: """ AgentBuilder can help user build an automatic task solving process powered by multi-agent system. - Specifically, our building pipeline include init(), build(), and start(). + Specifically, our building pipeline includes initialize and build. In build(), we prompt a gpt-4 model to create multiple participant agents, and specify whether this task need programming to solve. User can save the built agents' config by calling save(), and load the saved configs by load(), which can skip the @@ -107,11 +107,11 @@ def _create_agent( model_name_or_hf_repo: str, llm_config: dict, system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, - use_gpts: Optional[bool] = False, + use_oai_assistant: Optional[bool] = False, world_size: Optional[int] = 1, ) -> autogen.AssistantAgent: """ - Create group chat agents. + Create a group chat participant agent. If the agent rely on an open-source model, this function will automatically set up an endpoint for that agent. The API address of that endpoint will be "localhost:{free port}". @@ -121,7 +121,7 @@ def _create_agent( model_name_or_hf_repo: llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). system_message: system prompt use to format an agent's behavior. - use_gpts: use OpenAI GPTs api instead on self-construct agent. + use_oai_assistant: use OpenAI assistant api instead of self-constructed agent. world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs). Returns: @@ -190,7 +190,7 @@ def _create_agent( current_config.update( {"config_list": config_list, "model": model_name_or_hf_repo, "max_tokens": self.max_tokens} ) - if use_gpts: + if use_oai_assistant: from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent agent = GPTAssistantAgent( @@ -242,9 +242,8 @@ def build( default_llm_config: Optional[Dict] = None, coding: Optional[bool] = None, cached_configs: Optional[Dict] = None, - use_gpts: Optional[bool] = False, - user_proxy_work_dir: Optional[str] = None, - docker: Optional[Union[list, bool, str]] = False, + use_oai_assistant: Optional[bool] = False, + code_execution_config: Optional[Dict] = None, **kwargs, ): """ @@ -255,12 +254,19 @@ def build( default_llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). coding: use to identify if the user proxy (a code interpreter) should be added. cached_configs: previously saved agent configs. - use_gpts: if true, build function will use GPTs api to build agents. - user_proxy_work_dir: The working directory for the code execution. - docker (Optional, list, str or bool): The docker image to use for code execution. + use_oai_assistant: use OpenAI assistant api instead of self-constructed agent. + code_execution_config: specific configs for user proxy (e.g., last_n_messages, work_dir, ...). """ use_api = False + if code_execution_config is None: + code_execution_config = { + "last_n_messages": 2, + "work_dir": 'groupchat', + "use_docker": False, + "timeout": 60, + } + if cached_configs is None: use_api = True agent_configs = [] @@ -324,7 +330,7 @@ def build( config["model"], default_llm_config, system_message=config["system_message"], - use_gpts=use_gpts, + use_oai_assistant=use_oai_assistant, **kwargs, ) agent_list = [agent_config[0] for agent_config in self.agent_procs_assign.values()] @@ -346,12 +352,7 @@ def build( name="User_console_and_Python_code_interpreter", is_termination_msg=lambda x: "TERMINATE" in x.get("content"), system_message="User console with a python code interpreter interface.", - code_execution_config={ - "last_n_messages": 2, - "work_dir": user_proxy_work_dir, - "use_docker": docker, - "timeout": 60, - }, + code_execution_config=code_execution_config, human_input_mode="NEVER", ) ] + agent_list @@ -362,6 +363,7 @@ def build( "agent_configs": agent_configs, "coding": coding, "default_llm_config": default_llm_config, + "code_execution_config": code_execution_config, } ) diff --git a/notebook/agentchat_autobuild.ipynb b/notebook/agentchat_autobuild.ipynb index fab66c2ee59..02c592b5bb3 100644 --- a/notebook/agentchat_autobuild.ipynb +++ b/notebook/agentchat_autobuild.ipynb @@ -11,7 +11,7 @@ "AutoGen offers conversable agents powered by LLM, tool, or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, our main building pipeline include `build()` and `start()`. In `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. After that, user can call `start()` at a proper time to complete the task. AgentBuilder also support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html) and [Fastchat](https://github.com/lm-sys/FastChat). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html)." + "In this notebook, we introduce a new class, `AgentBuilder`, to help user build an automatic task solving process powered by multi-agent system. Specifically, in `build()`, we prompt a LLM to create multiple participant agent and initialize a group chat, and specify whether this task need programming to solve. AgentBuilder also support open-source LLMs by [vLLM](https://docs.vllm.ai/en/latest/index.html) and [Fastchat](https://github.com/lm-sys/FastChat). Check the supported model list [here](https://docs.vllm.ai/en/latest/models/supported_models.html)." ] }, { @@ -474,7 +474,6 @@ " \"system_message\": \"As a dedicated AI assistant for a software developer, your role involves employing your Python programming prowess and proficiency in natural language processing to facilitate the discovery and analysis of scholarly articles on arXiv. Your tasks include crafting Python scripts to automatically search, retrieve, and present information regarding the latest research, with a focus on applicable advancements in technology such as GPT-4 and its potential impact on the domain of software development.\\n\\n1. Utilize Python to programmatically seek out and extract pertinent data, for example, navigating or probing the web, downloading/ingesting documents, or showcasing content from web pages or files. When enough information has been accumulated to proceed, you will then analyze and interpret the findings.\\n\\n2. When there's a need to perform an operation programmatically, your Python code should accomplish the task and manifest the outcome. Progress through the task incrementally and systematically.\\n\\nProvide a clear plan outlining each stage of the task, specifying which components will be executed through Python coding and which through your linguistic capabilities. When proposing Python code, remember to:\\n\\n- Label the script type within the code block\\n- Avoid suggesting code that the user would need to alter\\n- Refrain from including more than one code block in your response\\n- Circumvent requesting the user to manually transcribe any results; utilize 'print' statements where applicable\\n- Examine the user's reported execution outcomes\\n\\nIf an error arises, your responsibility is to rectify the issue and submit the corrected script. Should an error remain unresolvable, or if the task remains incomplete post successful code execution, re-evaluate the scenario, gather any further required information, and formulate an alternative approach.\\n\\nUpon confirming that the task has been satisfactorily accomplished and the user's requirements have been met, indicate closure of the procedure with a concluding statement.\"\n", " }\n", " ],\n", - " \"manager_system_message\": \"Group chat manager.\",\n", " \"coding\": true,\n", " \"default_llm_config\": {\n", " \"temperature\": 0\n", @@ -872,10 +871,10 @@ "collapsed": false }, "source": [ - "## Use GPTs\n", + "## Use OpenAI Assistant\n", "\n", - "[GPTs](https://openai.com/blog/introducing-gpts) allow user to create an assistant with a simple instruction of the task. It has plugin support that can let ChatGPT complete some complex instructions, and can optionally update the assistant's instruction to let it adapted to new task or improve on the current task.\n", - "AutoBuild also support GPTs api by adding `use_gpts=True` to the `build()` function." + "[The Assistants API](https://platform.openai.com/docs/assistants/overview) allows you to build AI assistants within your own applications. An Assistant has instructions and can leverage models, tools, and knowledge to respond to user queries.\n", + "AutoBuild also support assistant api by adding `use_oai_assistant=True` to `build()`." ] }, { @@ -1101,7 +1100,7 @@ ], "source": [ "new_builder = AgentBuilder(config_path=config_path)\n", - "agent_list, agent_configs = new_builder.build(building_task, default_llm_config, use_gpts=True) # Transfer to GPTs API.\n", + "agent_list, agent_configs = new_builder.build(building_task, default_llm_config, use_oai_assistant=True) # Transfer to OpenAI assistant API.\n", "start_task(\n", " execution_task=\"Find a recent paper about XAI on arxiv and find its potential applications in medical.\",\n", " agent_list=agent_list,\n", From e38904d55dd76c35c828eb5abefbd3a1fdd361a7 Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Mon, 4 Dec 2023 10:05:13 +0800 Subject: [PATCH 73/74] recover contrib-openai.yml --- .github/workflows/contrib-openai.yml | 328 +++++++++++++-------------- 1 file changed, 164 insertions(+), 164 deletions(-) diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index 890e5a30bd1..467d5270c8e 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -4,7 +4,7 @@ name: OpenAI4ContribTests on: - pull_request: + pull_request_target: branches: ['main'] paths: - 'autogen/**' @@ -13,169 +13,169 @@ on: - 'setup.py' jobs: -# RetrieveChatTest: -# strategy: -# matrix: -# os: [ubuntu-latest] -# python-version: ["3.10"] -# runs-on: ${{ matrix.os }} -# environment: openai1 -# steps: -# # checkout to pr branch -# - name: Checkout -# uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha }} -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install packages and dependencies -# run: | -# docker --version -# python -m pip install --upgrade pip wheel -# pip install -e . -# python -c "import autogen" -# pip install coverage pytest-asyncio -# - name: Install packages for test when needed -# run: | -# pip install docker -# pip install qdrant_client[fastembed] -# pip install -e .[retrievechat] -# - name: Coverage -# env: -# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} -# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} -# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} -# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} -# run: | -# coverage run -a -m pytest test/agentchat/contrib/test_retrievechat.py test/agentchat/contrib/test_qdrant_retrievechat.py -# coverage xml -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v3 -# with: -# file: ./coverage.xml -# flags: unittests -# CompressionTest: -# strategy: -# matrix: -# os: [ubuntu-latest] -# python-version: ["3.9"] -# runs-on: ${{ matrix.os }} -# environment: openai1 -# steps: -# # checkout to pr branch -# - name: Checkout -# uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha }} -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install packages and dependencies -# run: | -# docker --version -# python -m pip install --upgrade pip wheel -# pip install -e . -# python -c "import autogen" -# pip install coverage pytest-asyncio -# - name: Install packages for test when needed -# run: | -# pip install docker -# - name: Coverage -# env: -# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} -# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} -# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} -# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} -# run: | -# coverage run -a -m pytest test/agentchat/contrib/test_compressible_agent.py -# coverage xml -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v3 -# with: -# file: ./coverage.xml -# flags: unittests -# GPTAssistantAgent: -# strategy: -# matrix: -# os: [ubuntu-latest] -# python-version: ["3.11"] -# runs-on: ${{ matrix.os }} -# environment: openai1 -# steps: -# # checkout to pr branch -# - name: Checkout -# uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha }} -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install packages and dependencies -# run: | -# docker --version -# python -m pip install --upgrade pip wheel -# pip install -e . -# python -c "import autogen" -# pip install coverage pytest-asyncio -# - name: Install packages for test when needed -# run: | -# pip install docker -# - name: Coverage -# env: -# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} -# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} -# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} -# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} -# run: | -# coverage run -a -m pytest test/agentchat/contrib/test_gpt_assistant.py -# coverage xml -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v3 -# with: -# file: ./coverage.xml -# flags: unittests -# TeachableAgent: -# strategy: -# matrix: -# os: [ubuntu-latest] -# python-version: ["3.11"] -# runs-on: ${{ matrix.os }} -# environment: openai1 -# steps: -# # checkout to pr branch -# - name: Checkout -# uses: actions/checkout@v3 -# with: -# ref: ${{ github.event.pull_request.head.sha }} -# - name: Set up Python ${{ matrix.python-version }} -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python-version }} -# - name: Install packages and dependencies -# run: | -# docker --version -# python -m pip install --upgrade pip wheel -# pip install -e .[teachable] -# python -c "import autogen" -# pip install coverage pytest -# - name: Coverage -# env: -# OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} -# AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} -# AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} -# OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} -# run: | -# coverage run -a -m pytest test/agentchat/contrib/test_teachable_agent.py -# coverage xml -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v3 -# with: -# file: ./coverage.xml -# flags: unittests + RetrieveChatTest: + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.10"] + runs-on: ${{ matrix.os }} + environment: openai1 + steps: + # checkout to pr branch + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies + run: | + docker --version + python -m pip install --upgrade pip wheel + pip install -e . + python -c "import autogen" + pip install coverage pytest-asyncio + - name: Install packages for test when needed + run: | + pip install docker + pip install qdrant_client[fastembed] + pip install -e .[retrievechat] + - name: Coverage + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + run: | + coverage run -a -m pytest test/agentchat/contrib/test_retrievechat.py test/agentchat/contrib/test_qdrant_retrievechat.py + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + CompressionTest: + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.9"] + runs-on: ${{ matrix.os }} + environment: openai1 + steps: + # checkout to pr branch + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies + run: | + docker --version + python -m pip install --upgrade pip wheel + pip install -e . + python -c "import autogen" + pip install coverage pytest-asyncio + - name: Install packages for test when needed + run: | + pip install docker + - name: Coverage + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + run: | + coverage run -a -m pytest test/agentchat/contrib/test_compressible_agent.py + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + GPTAssistantAgent: + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.11"] + runs-on: ${{ matrix.os }} + environment: openai1 + steps: + # checkout to pr branch + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies + run: | + docker --version + python -m pip install --upgrade pip wheel + pip install -e . + python -c "import autogen" + pip install coverage pytest-asyncio + - name: Install packages for test when needed + run: | + pip install docker + - name: Coverage + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + run: | + coverage run -a -m pytest test/agentchat/contrib/test_gpt_assistant.py + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + TeachableAgent: + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.11"] + runs-on: ${{ matrix.os }} + environment: openai1 + steps: + # checkout to pr branch + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies + run: | + docker --version + python -m pip install --upgrade pip wheel + pip install -e .[teachable] + python -c "import autogen" + pip install coverage pytest + - name: Coverage + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + run: | + coverage run -a -m pytest test/agentchat/contrib/test_teachable_agent.py + coverage xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests AgentBuilder: strategy: matrix: From bf6715bd978150c6986d14447db1e9a1a3d809c9 Mon Sep 17 00:00:00 2001 From: LinxinS97 Date: Mon, 4 Dec 2023 10:09:55 +0800 Subject: [PATCH 74/74] pre-commit solve --- autogen/agentchat/contrib/agent_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index 55590771a7d..50ef776c04c 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -262,7 +262,7 @@ def build( if code_execution_config is None: code_execution_config = { "last_n_messages": 2, - "work_dir": 'groupchat', + "work_dir": "groupchat", "use_docker": False, "timeout": 60, }