format

langchain-ai · Dec 18, 2023 · 9ffb257 · 9ffb257
1 parent edc9b21
commit 9ffb257
Showing 1 changed file with 157 additions and 11 deletions.
diff --git a/docs/source/notebooks/tool_usage/benchmark_all_tasks.ipynb b/docs/source/notebooks/tool_usage/benchmark_all_tasks.ipynb
@@ -15,6 +15,19 @@
   {
    "cell_type": "code",
    "execution_count": null,
+   "id": "62bc0480-88b0-4d66-aba1-1214ee7f8af3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%env LANGCHAIN_API_KEY=ls__79d58f8843574c68b8a722157426870b\n",
+    "%env LANGCHAIN_ENDPOINT=https://dev.api.smith.langchain.com"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
    "id": "13a7483b-d08f-49fa-83da-619863171e5b",
    "metadata": {
     "tags": []
@@ -36,7 +49,7 @@
     "    AnthropicToolUserFactory,\n",
     "    CustomAgentFactory,\n",
     "    OpenAIAgentFactory,\n",
-    "    OpenAIAssistantFactory\n",
+    "    OpenAIAssistantFactory,\n",
     ")"
    ]
   },
@@ -51,30 +64,159 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "e0f2bb0c-c741-4fb4-96bc-54b3ee88bf5b",
    "metadata": {
     "tags": []
    },
    "outputs": [],
    "source": [
-    "task = registry[\"Multiverse Math\"]"
+    "task = registry[\"Tool Usage - Typewriter (26 tools)\"]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
+   "id": "6326156d-416c-43fd-93a2-c9678bfc4626",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "542dc329-0358-4fe7-903b-3569566448af",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import tiktoken\n",
+    "\n",
+    "encoder = tiktoken.encoding_for_model(\"gpt-3.5-turbo\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a0d3eb48-5cea-4eed-b01a-1806df639fe5",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(encoder.encode(\"abc cba\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
    "id": "adfbcaa9-349c-4223-89be-4abff9cf76ff",
    "metadata": {
     "tags": []
    },
    "outputs": [],
    "source": [
     "agent_factory = OpenAIAgentFactory(\n",
-    "    task, model=\"mistral-7b-instruct-v0.1\"\n",
+    "    task, model=\"gpt-3.5-turbo-1106\"\n",
     ")  # Follows OpenAI function format\n",
     "# agent_factory = OpenAIAgentFactory(task, model='gpt-3.5-turbo-1106')\n",
-    "agent_factory().invoke({\"question\": \"(2 + 5) and then to the power of 0.5\"})"
+    "from concurrent.futures import ThreadPoolExecutor\n",
+    "\n",
+    "\n",
+    "def run_(_):\n",
+    "    try:\n",
+    "        return agent_factory().invoke({\"question\": \"dic\"})\n",
+    "    except Exception as e:\n",
+    "        return e\n",
+    "\n",
+    "\n",
+    "with ThreadPoolExecutor() as executor:\n",
+    "    results = list(executor.map(run_, list(range(50))))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "ccad7131-ff84-4f58-b429-0549e084afef",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "47"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sum([isinstance(v, Exception) for v in results])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "3cdc3240-3da3-4cd1-963b-b7944876218b",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "agent = agent_factory()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "d8004f5a-4eb2-40a3-a769-5291c526511f",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "RunnableBinding(bound=RunnableLambda(...), config={'run_name': 'Format Input'})\n",
+       "| AgentExecutor(agent=RunnableMultiActionAgent(runnable={\n",
+       "    input: RunnableLambda(...),\n",
+       "    agent_scratchpad: RunnableLambda(...)\n",
+       "  }\n",
+       "  | ChatPromptTemplate(input_variables=['agent_scratchpad', 'input'], input_types={'agent_scratchpad': typing.List[typing.Union[langchain_core.messages.ai.AIMessage, langchain_core.messages.human.HumanMessage, langchain_core.messages.chat.ChatMessage, langchain_core.messages.system.SystemMessage, langchain_core.messages.function.FunctionMessage, langchain_core.messages.tool.ToolMessage]]}, messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\")), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['input'], template='{input}')), MessagesPlaceholder(variable_name='agent_scratchpad')])\n",
+       "  | RunnableBinding(bound=ChatOpenAI(client=<openai.resources.chat.completions.Completions object at 0x2ad3e0350>, async_client=<openai.resources.chat.completions.AsyncCompletions object at 0x2ae2b4a90>, model_name='gpt-3.5-turbo-1106', temperature=0.0, model_kwargs={'seed': 0}, openai_api_key='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', openai_proxy=''), kwargs={'tools': [{'type': 'function', 'function': {'name': 'a', 'description': 'a() -> str - Run to Type the letter \"a\".', 'parameters': {'title': 'aSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'b', 'description': 'b() -> str - Run to Type the letter \"b\".', 'parameters': {'title': 'bSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'c', 'description': 'c() -> str - Run to Type the letter \"c\".', 'parameters': {'title': 'cSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'd', 'description': 'd() -> str - Run to Type the letter \"d\".', 'parameters': {'title': 'dSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'e', 'description': 'e() -> str - Run to Type the letter \"e\".', 'parameters': {'title': 'eSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'f', 'description': 'f() -> str - Run to Type the letter \"f\".', 'parameters': {'title': 'fSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'g', 'description': 'g() -> str - Run to Type the letter \"g\".', 'parameters': {'title': 'gSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'h', 'description': 'h() -> str - Run to Type the letter \"h\".', 'parameters': {'title': 'hSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'i', 'description': 'i() -> str - Run to Type the letter \"i\".', 'parameters': {'title': 'iSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'j', 'description': 'j() -> str - Run to Type the letter \"j\".', 'parameters': {'title': 'jSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'k', 'description': 'k() -> str - Run to Type the letter \"k\".', 'parameters': {'title': 'kSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'l', 'description': 'l() -> str - Run to Type the letter \"l\".', 'parameters': {'title': 'lSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'm', 'description': 'm() -> str - Run to Type the letter \"m\".', 'parameters': {'title': 'mSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'n', 'description': 'n() -> str - Run to Type the letter \"n\".', 'parameters': {'title': 'nSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'o', 'description': 'o() -> str - Run to Type the letter \"o\".', 'parameters': {'title': 'oSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'p', 'description': 'p() -> str - Run to Type the letter \"p\".', 'parameters': {'title': 'pSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'q', 'description': 'q() -> str - Run to Type the letter \"q\".', 'parameters': {'title': 'qSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'r', 'description': 'r() -> str - Run to Type the letter \"r\".', 'parameters': {'title': 'rSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 's', 'description': 's() -> str - Run to Type the letter \"s\".', 'parameters': {'title': 'sSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 't', 'description': 't() -> str - Run to Type the letter \"t\".', 'parameters': {'title': 'tSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'u', 'description': 'u() -> str - Run to Type the letter \"u\".', 'parameters': {'title': 'uSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'v', 'description': 'v() -> str - Run to Type the letter \"v\".', 'parameters': {'title': 'vSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'w', 'description': 'w() -> str - Run to Type the letter \"w\".', 'parameters': {'title': 'wSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'x', 'description': 'x() -> str - Run to Type the letter \"x\".', 'parameters': {'title': 'xSchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'y', 'description': 'y() -> str - Run to Type the letter \"y\".', 'parameters': {'title': 'ySchemaSchema', 'type': 'object', 'properties': {}}}}, {'type': 'function', 'function': {'name': 'z', 'description': 'z() -> str - Run to Type the letter \"z\".', 'parameters': {'title': 'zSchemaSchema', 'type': 'object', 'properties': {}}}}]})\n",
+       "  | OpenAIToolsAgentOutputParser()), tools=[StructuredTool(name='a', description='a() -> str - Run to Type the letter \"a\".', args_schema=<class 'pydantic.v1.main.aSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae2845e0>), StructuredTool(name='b', description='b() -> str - Run to Type the letter \"b\".', args_schema=<class 'pydantic.v1.main.bSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284680>), StructuredTool(name='c', description='c() -> str - Run to Type the letter \"c\".', args_schema=<class 'pydantic.v1.main.cSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284720>), StructuredTool(name='d', description='d() -> str - Run to Type the letter \"d\".', args_schema=<class 'pydantic.v1.main.dSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae2847c0>), StructuredTool(name='e', description='e() -> str - Run to Type the letter \"e\".', args_schema=<class 'pydantic.v1.main.eSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284860>), StructuredTool(name='f', description='f() -> str - Run to Type the letter \"f\".', args_schema=<class 'pydantic.v1.main.fSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284900>), StructuredTool(name='g', description='g() -> str - Run to Type the letter \"g\".', args_schema=<class 'pydantic.v1.main.gSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae2849a0>), StructuredTool(name='h', description='h() -> str - Run to Type the letter \"h\".', args_schema=<class 'pydantic.v1.main.hSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284a40>), StructuredTool(name='i', description='i() -> str - Run to Type the letter \"i\".', args_schema=<class 'pydantic.v1.main.iSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284ae0>), StructuredTool(name='j', description='j() -> str - Run to Type the letter \"j\".', args_schema=<class 'pydantic.v1.main.jSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284b80>), StructuredTool(name='k', description='k() -> str - Run to Type the letter \"k\".', args_schema=<class 'pydantic.v1.main.kSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284c20>), StructuredTool(name='l', description='l() -> str - Run to Type the letter \"l\".', args_schema=<class 'pydantic.v1.main.lSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284cc0>), StructuredTool(name='m', description='m() -> str - Run to Type the letter \"m\".', args_schema=<class 'pydantic.v1.main.mSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284d60>), StructuredTool(name='n', description='n() -> str - Run to Type the letter \"n\".', args_schema=<class 'pydantic.v1.main.nSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284e00>), StructuredTool(name='o', description='o() -> str - Run to Type the letter \"o\".', args_schema=<class 'pydantic.v1.main.oSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284ea0>), StructuredTool(name='p', description='p() -> str - Run to Type the letter \"p\".', args_schema=<class 'pydantic.v1.main.pSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284f40>), StructuredTool(name='q', description='q() -> str - Run to Type the letter \"q\".', args_schema=<class 'pydantic.v1.main.qSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae284fe0>), StructuredTool(name='r', description='r() -> str - Run to Type the letter \"r\".', args_schema=<class 'pydantic.v1.main.rSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae285080>), StructuredTool(name='s', description='s() -> str - Run to Type the letter \"s\".', args_schema=<class 'pydantic.v1.main.sSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae285120>), StructuredTool(name='t', description='t() -> str - Run to Type the letter \"t\".', args_schema=<class 'pydantic.v1.main.tSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae2851c0>), StructuredTool(name='u', description='u() -> str - Run to Type the letter \"u\".', args_schema=<class 'pydantic.v1.main.uSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae285260>), StructuredTool(name='v', description='v() -> str - Run to Type the letter \"v\".', args_schema=<class 'pydantic.v1.main.vSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae285300>), StructuredTool(name='w', description='w() -> str - Run to Type the letter \"w\".', args_schema=<class 'pydantic.v1.main.wSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae2853a0>), StructuredTool(name='x', description='x() -> str - Run to Type the letter \"x\".', args_schema=<class 'pydantic.v1.main.xSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae285440>), StructuredTool(name='y', description='y() -> str - Run to Type the letter \"y\".', args_schema=<class 'pydantic.v1.main.ySchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae2854e0>), StructuredTool(name='z', description='z() -> str - Run to Type the letter \"z\".', args_schema=<class 'pydantic.v1.main.zSchemaSchema'>, func=<function _create_typing_func.<locals>.func at 0x2ae285580>)], return_intermediate_steps=True, handle_parsing_errors=True)\n",
+       "| RunnableBinding(bound=RunnableLambda(...), config={'run_name': 'Ensure Output'})\n",
+       "| RunnableBinding(bound=RunnableAssign(mapper={\n",
+       "    state: RunnableLambda(...)\n",
+       "  }), config={'run_name': 'Read Env State'})"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "agent"
    ]
   },
   {
@@ -116,14 +258,14 @@
    "source": [
     "tests = [\n",
     "    # 2-tuple of (architecture, model name)\n",
-    "    (\"anthropic_tool_user\", \"claude-2.1\"),\n",
-    "    (\"openai_functions\", \"mistral-7b-instruct-v0.1\"),\n",
+    "    # (\"anthropic_tool_user\", \"claude-2.1\"),\n",
+    "    # (\"openai_functions\", \"mistral-7b-instruct-v0.1\"),\n",
     "    (\"openai_functions\", \"gpt-3.5-turbo-1106\"),\n",
     "    (\"openai_functions\", \"gpt-3.5-turbo-0613\"),\n",
     "    (\"openai_functions\", \"gpt-4-1106-preview\"),\n",
     "    (\"openai_functions\", \"gpt-4-0613\"),\n",
-    "    (\"openai_assistant\", \"gpt-4-1106-preview\"),\n",
-    "    (\"openai_assistant\", \"gpt-3.5-turbo-1106\")\n",
+    "    # (\"openai_assistant\", \"gpt-4-1106-preview\"),\n",
+    "    # (\"openai_assistant\", \"gpt-3.5-turbo-1106\")\n",
     "]"
    ]
   },
@@ -163,6 +305,8 @@
     "for task in registry:\n",
     "    if task.type != \"ToolUsageTask\":\n",
     "        continue\n",
+    "    if \"26\" not in task.name:\n",
+    "        continue\n",
     "\n",
     "    dataset_name = task.name\n",
     "    clone_public_dataset(task.dataset_id, dataset_name=dataset_name)\n",
@@ -362,7 +506,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "69df66a1-960c-40a3-abc8-58b503fceda5",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "x = np.arange(len(tasks))  # the label locations\n",