From 9ffb2571c5c08ee535b245648ce4999b4c61ebfd Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:26:11 -0800 Subject: [PATCH] format --- .../tool_usage/benchmark_all_tasks.ipynb | 168 ++++++++++++++++-- 1 file changed, 157 insertions(+), 11 deletions(-) diff --git a/docs/source/notebooks/tool_usage/benchmark_all_tasks.ipynb b/docs/source/notebooks/tool_usage/benchmark_all_tasks.ipynb index c043ed6..2026ce7 100644 --- a/docs/source/notebooks/tool_usage/benchmark_all_tasks.ipynb +++ b/docs/source/notebooks/tool_usage/benchmark_all_tasks.ipynb @@ -15,6 +15,19 @@ { "cell_type": "code", "execution_count": null, + "id": "62bc0480-88b0-4d66-aba1-1214ee7f8af3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%env LANGCHAIN_API_KEY=ls__79d58f8843574c68b8a722157426870b\n", + "%env LANGCHAIN_ENDPOINT=https://dev.api.smith.langchain.com" + ] + }, + { + "cell_type": "code", + "execution_count": 1, "id": "13a7483b-d08f-49fa-83da-619863171e5b", "metadata": { "tags": [] @@ -36,7 +49,7 @@ " AnthropicToolUserFactory,\n", " CustomAgentFactory,\n", " OpenAIAgentFactory,\n", - " OpenAIAssistantFactory\n", + " OpenAIAssistantFactory,\n", ")" ] }, @@ -51,19 +64,68 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "e0f2bb0c-c741-4fb4-96bc-54b3ee88bf5b", "metadata": { "tags": [] }, "outputs": [], "source": [ - "task = registry[\"Multiverse Math\"]" + "task = registry[\"Tool Usage - Typewriter (26 tools)\"]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, + "id": "6326156d-416c-43fd-93a2-c9678bfc4626", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.chat_models import ChatOpenAI" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "542dc329-0358-4fe7-903b-3569566448af", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import tiktoken\n", + "\n", + "encoder = tiktoken.encoding_for_model(\"gpt-3.5-turbo\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "a0d3eb48-5cea-4eed-b01a-1806df639fe5", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "3" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(encoder.encode(\"abc cba\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "id": "adfbcaa9-349c-4223-89be-4abff9cf76ff", "metadata": { "tags": [] @@ -71,10 +133,90 @@ "outputs": [], "source": [ "agent_factory = OpenAIAgentFactory(\n", - " task, model=\"mistral-7b-instruct-v0.1\"\n", + " task, model=\"gpt-3.5-turbo-1106\"\n", ") # Follows OpenAI function format\n", "# agent_factory = OpenAIAgentFactory(task, model='gpt-3.5-turbo-1106')\n", - "agent_factory().invoke({\"question\": \"(2 + 5) and then to the power of 0.5\"})" + "from concurrent.futures import ThreadPoolExecutor\n", + "\n", + "\n", + "def run_(_):\n", + " try:\n", + " return agent_factory().invoke({\"question\": \"dic\"})\n", + " except Exception as e:\n", + " return e\n", + "\n", + "\n", + "with ThreadPoolExecutor() as executor:\n", + " results = list(executor.map(run_, list(range(50))))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "ccad7131-ff84-4f58-b429-0549e084afef", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "47" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum([isinstance(v, Exception) for v in results])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "3cdc3240-3da3-4cd1-963b-b7944876218b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "agent = agent_factory()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "d8004f5a-4eb2-40a3-a769-5291c526511f", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "RunnableBinding(bound=RunnableLambda(...), config={'run_name': 'Format Input'})\n", + "| AgentExecutor(agent=RunnableMultiActionAgent(runnable={\n", + " input: RunnableLambda(...),\n", + " agent_scratchpad: RunnableLambda(...)\n", + " }\n", + " | ChatPromptTemplate(input_variables=['agent_scratchpad', 'input'], input_types={'agent_scratchpad': typing.List[typing.Union[langchain_core.messages.ai.AIMessage, langchain_core.messages.human.HumanMessage, langchain_core.messages.chat.ChatMessage, langchain_core.messages.system.SystemMessage, langchain_core.messages.function.FunctionMessage, langchain_core.messages.tool.ToolMessage]]}, messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\")), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['input'], template='{input}')), MessagesPlaceholder(variable_name='agent_scratchpad')])\n", + " | RunnableBinding(bound=ChatOpenAI(client=, async_client=, model_name='gpt-3.5-turbo-1106', temperature=0.0, model_kwargs={'seed': 0}, openai_api_key='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', openai_proxy=''), kwargs={'tools': [{'type': 'function', 'function': {'name': 'a', 'description': 'a() -> str - Run to Type the letter \"a\".', 'parameters': {'title': 'aSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'b', 'description': 'b() -> str - Run to Type the letter \"b\".', 'parameters': {'title': 'bSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'c', 'description': 'c() -> str - Run to Type the letter \"c\".', 'parameters': {'title': 'cSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'd', 'description': 'd() -> str - Run to Type the letter \"d\".', 'parameters': {'title': 'dSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'e', 'description': 'e() -> str - Run to Type the letter \"e\".', 'parameters': {'title': 'eSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'f', 'description': 'f() -> str - Run to Type the letter \"f\".', 'parameters': {'title': 'fSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'g', 'description': 'g() -> str - Run to Type the letter \"g\".', 'parameters': {'title': 'gSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'h', 'description': 'h() -> str - Run to Type the letter \"h\".', 'parameters': {'title': 'hSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'i', 'description': 'i() -> str - Run to Type the letter \"i\".', 'parameters': {'title': 'iSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'j', 'description': 'j() -> str - Run to Type the letter \"j\".', 'parameters': {'title': 'jSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'k', 'description': 'k() -> str - Run to Type the letter \"k\".', 'parameters': {'title': 'kSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'l', 'description': 'l() -> str - Run to Type the letter \"l\".', 'parameters': {'title': 'lSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'm', 'description': 'm() -> str - Run to Type the letter \"m\".', 'parameters': {'title': 'mSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'n', 'description': 'n() -> str - Run to Type the letter \"n\".', 'parameters': {'title': 'nSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'o', 'description': 'o() -> str - Run to Type the letter \"o\".', 'parameters': {'title': 'oSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'p', 'description': 'p() -> str - Run to Type the letter \"p\".', 'parameters': {'title': 'pSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'q', 'description': 'q() -> str - Run to Type the letter \"q\".', 'parameters': {'title': 'qSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'r', 'description': 'r() -> str - Run to Type the letter \"r\".', 'parameters': {'title': 'rSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 's', 'description': 's() -> str - Run to Type the letter \"s\".', 'parameters': {'title': 'sSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 't', 'description': 't() -> str - Run to Type the letter \"t\".', 'parameters': {'title': 'tSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'u', 'description': 'u() -> str - Run to Type the letter \"u\".', 'parameters': {'title': 'uSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'v', 'description': 'v() -> str - Run to Type the letter \"v\".', 'parameters': {'title': 'vSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'w', 'description': 'w() -> str - Run to Type the letter \"w\".', 'parameters': {'title': 'wSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'x', 'description': 'x() -> str - Run to Type the letter \"x\".', 'parameters': {'title': 'xSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'y', 'description': 'y() -> str - Run to Type the letter \"y\".', 'parameters': {'title': 'ySchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'z', 'description': 'z() -> str - Run to Type the letter \"z\".', 'parameters': {'title': 'zSchemaSchema', 'type': 'object', 'properties': {}}}}]})\n", + " | OpenAIToolsAgentOutputParser()), tools=[StructuredTool(name='a', description='a() -> str - Run to Type the letter \"a\".', args_schema=, func=.func at 0x2ae2845e0>), StructuredTool(name='b', description='b() -> str - Run to Type the letter \"b\".', args_schema=, func=.func at 0x2ae284680>), StructuredTool(name='c', description='c() -> str - Run to Type the letter \"c\".', args_schema=, func=.func at 0x2ae284720>), StructuredTool(name='d', description='d() -> str - Run to Type the letter \"d\".', args_schema=, func=.func at 0x2ae2847c0>), StructuredTool(name='e', description='e() -> str - Run to Type the letter \"e\".', args_schema=, func=.func at 0x2ae284860>), StructuredTool(name='f', description='f() -> str - Run to Type the letter \"f\".', args_schema=, func=.func at 0x2ae284900>), StructuredTool(name='g', description='g() -> str - Run to Type the letter \"g\".', args_schema=, func=.func at 0x2ae2849a0>), StructuredTool(name='h', description='h() -> str - Run to Type the letter \"h\".', args_schema=, func=.func at 0x2ae284a40>), StructuredTool(name='i', description='i() -> str - Run to Type the letter \"i\".', args_schema=, func=.func at 0x2ae284ae0>), StructuredTool(name='j', description='j() -> str - Run to Type the letter \"j\".', args_schema=, func=.func at 0x2ae284b80>), StructuredTool(name='k', description='k() -> str - Run to Type the letter \"k\".', args_schema=, func=.func at 0x2ae284c20>), StructuredTool(name='l', description='l() -> str - Run to Type the letter \"l\".', args_schema=, func=.func at 0x2ae284cc0>), StructuredTool(name='m', description='m() -> str - Run to Type the letter \"m\".', args_schema=, func=.func at 0x2ae284d60>), StructuredTool(name='n', description='n() -> str - Run to Type the letter \"n\".', args_schema=, func=.func at 0x2ae284e00>), StructuredTool(name='o', description='o() -> str - Run to Type the letter \"o\".', args_schema=, func=.func at 0x2ae284ea0>), StructuredTool(name='p', description='p() -> str - Run to Type the letter \"p\".', args_schema=, func=.func at 0x2ae284f40>), StructuredTool(name='q', description='q() -> str - Run to Type the letter \"q\".', args_schema=, func=.func at 0x2ae284fe0>), StructuredTool(name='r', description='r() -> str - Run to Type the letter \"r\".', args_schema=, func=.func at 0x2ae285080>), StructuredTool(name='s', description='s() -> str - Run to Type the letter \"s\".', args_schema=, func=.func at 0x2ae285120>), StructuredTool(name='t', description='t() -> str - Run to Type the letter \"t\".', args_schema=, func=.func at 0x2ae2851c0>), StructuredTool(name='u', description='u() -> str - Run to Type the letter \"u\".', args_schema=, func=.func at 0x2ae285260>), StructuredTool(name='v', description='v() -> str - Run to Type the letter \"v\".', args_schema=, func=.func at 0x2ae285300>), StructuredTool(name='w', description='w() -> str - Run to Type the letter \"w\".', args_schema=, func=.func at 0x2ae2853a0>), StructuredTool(name='x', description='x() -> str - Run to Type the letter \"x\".', args_schema=, func=.func at 0x2ae285440>), StructuredTool(name='y', description='y() -> str - Run to Type the letter \"y\".', args_schema=, func=.func at 0x2ae2854e0>), StructuredTool(name='z', description='z() -> str - Run to Type the letter \"z\".', args_schema=, func=.func at 0x2ae285580>)], return_intermediate_steps=True, handle_parsing_errors=True)\n", + "| RunnableBinding(bound=RunnableLambda(...), config={'run_name': 'Ensure Output'})\n", + "| RunnableBinding(bound=RunnableAssign(mapper={\n", + " state: RunnableLambda(...)\n", + " }), config={'run_name': 'Read Env State'})" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agent" ] }, { @@ -116,14 +258,14 @@ "source": [ "tests = [\n", " # 2-tuple of (architecture, model name)\n", - " (\"anthropic_tool_user\", \"claude-2.1\"),\n", - " (\"openai_functions\", \"mistral-7b-instruct-v0.1\"),\n", + " # (\"anthropic_tool_user\", \"claude-2.1\"),\n", + " # (\"openai_functions\", \"mistral-7b-instruct-v0.1\"),\n", " (\"openai_functions\", \"gpt-3.5-turbo-1106\"),\n", " (\"openai_functions\", \"gpt-3.5-turbo-0613\"),\n", " (\"openai_functions\", \"gpt-4-1106-preview\"),\n", " (\"openai_functions\", \"gpt-4-0613\"),\n", - " (\"openai_assistant\", \"gpt-4-1106-preview\"),\n", - " (\"openai_assistant\", \"gpt-3.5-turbo-1106\")\n", + " # (\"openai_assistant\", \"gpt-4-1106-preview\"),\n", + " # (\"openai_assistant\", \"gpt-3.5-turbo-1106\")\n", "]" ] }, @@ -163,6 +305,8 @@ "for task in registry:\n", " if task.type != \"ToolUsageTask\":\n", " continue\n", + " if \"26\" not in task.name:\n", + " continue\n", "\n", " dataset_name = task.name\n", " clone_public_dataset(task.dataset_id, dataset_name=dataset_name)\n", @@ -362,7 +506,9 @@ "cell_type": "code", "execution_count": null, "id": "69df66a1-960c-40a3-abc8-58b503fceda5", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ "x = np.arange(len(tasks)) # the label locations\n",