Skip to content

Commit

Permalink
fixing import ordering
Browse files Browse the repository at this point in the history
  • Loading branch information
@jluey1
jluey1 committed Apr 29, 2024
1 parent 9a6ecfa commit be35fbc
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 72 deletions.
2 changes: 1 addition & 1 deletion autogen/agentchat/contrib/agent_eval/agent_eval.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
from typing import Dict, List, Optional, Union

import autogen
from autogen.agentchat.contrib.agent_eval.criterion import Criterion
from autogen.agentchat.contrib.agent_eval.critic_agent import CriticAgent
from autogen.agentchat.contrib.agent_eval.quantifier_agent import QuantifierAgent
from autogen.agentchat.contrib.agent_eval.subcritic_agent import SubCriticAgent
from autogen.agentchat.contrib.agent_eval.criterion import Criterion
from autogen.agentchat.contrib.agent_eval.task import Task


Expand Down
107 changes: 36 additions & 71 deletions notebook/agenteval_cq_math.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,27 +57,27 @@
"Defaulting to user installation because normal site-packages is not writeable\n",
"Requirement already satisfied: pyautogen>=0.2.3 in /home/vscode/.local/lib/python3.10/site-packages (0.2.17)\n",
"Requirement already satisfied: docker in /home/vscode/.local/lib/python3.10/site-packages (7.0.0)\n",
"Requirement already satisfied: termcolor in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.4.0)\n",
"Requirement already satisfied: python-dotenv in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.0.1)\n",
"Requirement already satisfied: pydantic!=2.6.0,<3,>=1.10 in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.6.4)\n",
"Requirement already satisfied: tiktoken in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (0.6.0)\n",
"Requirement already satisfied: termcolor in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.4.0)\n",
"Requirement already satisfied: diskcache in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (5.6.3)\n",
"Requirement already satisfied: flaml in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.1.2)\n",
"Requirement already satisfied: openai>=1.3 in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.14.1)\n",
"Requirement already satisfied: diskcache in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (5.6.3)\n",
"Requirement already satisfied: python-dotenv in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.0.1)\n",
"Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/site-packages (from docker) (2.2.1)\n",
"Requirement already satisfied: packaging>=14.0 in /usr/local/lib/python3.10/site-packages (from docker) (24.0)\n",
"Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/site-packages (from docker) (2.31.0)\n",
"Requirement already satisfied: tqdm>4 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.66.2)\n",
"Requirement already satisfied: httpx<1,>=0.23.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (0.27.0)\n",
"Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/site-packages (from docker) (2.2.1)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (1.9.0)\n",
"Requirement already satisfied: sniffio in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (1.3.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.7 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.10.0)\n",
"Requirement already satisfied: anyio<5,>=3.5.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.3.0)\n",
"Requirement already satisfied: annotated-types>=0.4.0 in /home/vscode/.local/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3) (0.6.0)\n",
"Requirement already satisfied: httpx<1,>=0.23.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (0.27.0)\n",
"Requirement already satisfied: typing-extensions<5,>=4.7 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.10.0)\n",
"Requirement already satisfied: sniffio in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (1.3.1)\n",
"Requirement already satisfied: tqdm>4 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.66.2)\n",
"Requirement already satisfied: pydantic-core==2.16.3 in /home/vscode/.local/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3) (2.16.3)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->docker) (2024.2.2)\n",
"Requirement already satisfied: annotated-types>=0.4.0 in /home/vscode/.local/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3) (0.6.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->docker) (3.6)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->docker) (3.3.2)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->docker) (2024.2.2)\n",
"Requirement already satisfied: NumPy>=1.17 in /home/vscode/.local/lib/python3.10/site-packages (from flaml->pyautogen>=0.2.3) (1.26.4)\n",
"Requirement already satisfied: regex>=2022.1.18 in /home/vscode/.local/lib/python3.10/site-packages (from tiktoken->pyautogen>=0.2.3) (2023.12.25)\n",
"Requirement already satisfied: exceptiongroup>=1.0.2 in /home/vscode/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai>=1.3->pyautogen>=0.2.3) (1.2.0)\n",
Expand All @@ -96,15 +96,15 @@
"Note: you may need to restart the kernel to use updated packages.\n",
"Defaulting to user installation because normal site-packages is not writeable\n",
"Requirement already satisfied: matplotlib in /home/vscode/.local/lib/python3.10/site-packages (3.8.3)\n",
"Requirement already satisfied: pyparsing>=2.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (3.1.2)\n",
"Requirement already satisfied: numpy<2,>=1.21 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.26.4)\n",
"Requirement already satisfied: kiwisolver>=1.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.4.5)\n",
"Requirement already satisfied: pillow>=8 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (10.2.0)\n",
"Requirement already satisfied: contourpy>=1.0.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.2.0)\n",
"Requirement already satisfied: python-dateutil>=2.7 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (2.9.0.post0)\n",
"Requirement already satisfied: cycler>=0.10 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (0.12.1)\n",
"Requirement already satisfied: contourpy>=1.0.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.2.0)\n",
"Requirement already satisfied: fonttools>=4.22.0 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (4.50.0)\n",
"Requirement already satisfied: pyparsing>=2.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (3.1.2)\n",
"Requirement already satisfied: pillow>=8 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (10.2.0)\n",
"Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib) (24.0)\n",
"Requirement already satisfied: numpy<2,>=1.21 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.26.4)\n",
"Requirement already satisfied: python-dateutil>=2.7 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (2.9.0.post0)\n",
"Requirement already satisfied: kiwisolver>=1.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.4.5)\n",
"Requirement already satisfied: six>=1.5 in /home/vscode/.local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
"\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
Expand Down Expand Up @@ -137,33 +137,17 @@
"metadata": {
"id": "YRycFEDJ1_BJ"
},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'critic_agent'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[2], line 13\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mautogen\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magentchat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontrib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magent_eval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcritic_agent\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m CriticAgent\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mautogen\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magentchat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontrib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magent_eval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mquantifier_agent\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m QuantifierAgent\n\u001b[0;32m---> 13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mautogen\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magentchat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontrib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magent_eval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magent_eval\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m generate_criteria, quantify_criteria\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mautogen\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magentchat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontrib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magent_eval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcriterion\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Criterion\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mautogen\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magentchat\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontrib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01magent_eval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtask\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Task\n",
"File \u001b[0;32m/workspaces/fork_autogen/autogen/agentchat/contrib/agent_eval/agent_eval.py:6\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Callable, Dict, List, Optional, Union\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mautogen\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mcritic_agent\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m CriticAgent\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mquantifier_agent\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m QuantifierAgent\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msubcritic_agent\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SubCriticAgent\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'critic_agent'"
]
}
],
"outputs": [],
"source": [
"import json\n",
"import os\n",
"import sys\n",
"from pathlib import Path\n",
"\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import scipy.stats as stats\n",
"\n",
"import autogen\n",
"from autogen.agentchat.contrib.agent_eval.critic_agent import CriticAgent\n",
"from autogen.agentchat.contrib.agent_eval.quantifier_agent import QuantifierAgent\n",
"from autogen.agentchat.contrib.agent_eval.agent_eval import generate_criteria, quantify_criteria\n",
"from autogen.agentchat.contrib.agent_eval.criterion import Criterion\n",
"from autogen.agentchat.contrib.agent_eval.task import Task\n",
Expand All @@ -184,7 +168,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {
"id": "5H1WRs_wkiK0"
},
Expand All @@ -194,13 +178,7 @@
"output_type": "stream",
"text": [
"\u001b[33mcritic_user\u001b[0m (to chat_manager):\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Task: Math problem solving.\n",
" Task description: Given any question, the system needs to solve the problem as consisely and accurately as possible\n",
" Task successful example: {'problem': 'What is the sum of all the distinct positive two-digit factors of 144?', 'level': 'Level 5', 'type': 'Number Theory', 'solution': 'Prime factorize $144=2^4\\\\cdot3^2$. The sum of the positive two-digit factors of 144 is $2^4+2\\\\cdot3^2+2^2\\\\cdot3+2^2\\\\cdot3^2+2^3\\\\cdot3+2^3\\\\cdot3^2+2^4\\\\cdot3=\\\\boxed{226}.$', 'problem_id': '0', 'response_with_ans': 'To find the sum of all the distinct positive two-digit factors of 144, we need to first find all these factors. We can do this by iterating through the numbers from 10 to 99 and checking if they are factors of 144. Then, we can sum these factors and print their sum.\\n\\nHere\\'s a Python script to accomplish this:\\n\\n```python\\ntwo_digit_factors = []\\n\\nfor i in range(10, 100):\\n if 144 % i == 0:\\n two_digit_factors.append(i)\\n\\nsum_of_factors = sum(two_digit_factors)\\nprint(\"The sum of all the distinct positive two-digit factors of 144 is:\", sum_of_factors)\\n```\\n\\nPlease run this script to find the desired sum.', 'round': 0, 'messages': [{'content': 'What is the sum of all the distinct positive two-digit factors of 144?', 'role': 'user'}, {'content': 'To find the sum of all the distinct positive two-digit factors of 144, we need to first find all these factors. We can do this by iterating through the numbers from 10 to 99 and checking if they are factors of 144. Then, we can sum these factors and print their sum.\\n\\nHere\\'s a Python script to accomplish this:\\n\\n```python\\ntwo_digit_factors = []\\n\\nfor i in range(10, 100):\\n if 144 % i == 0:\\n two_digit_factors.append(i)\\n\\nsum_of_factors = sum(two_digit_factors)\\nprint(\"The sum of all the distinct positive two-digit factors of 144 is:\", sum_of_factors)\\n```\\n\\nPlease run this script to find the desired sum.', 'role': 'assistant'}], 'time': 11.140539407730103, 'trial': -1}\n",
Expand Down Expand Up @@ -317,7 +295,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
Expand Down Expand Up @@ -356,7 +334,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {
"id": "4uUkZJh_subA"
},
Expand Down Expand Up @@ -385,7 +363,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
Expand Down Expand Up @@ -528,35 +506,28 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"--2024-04-17 22:07:57-- https://github.com/julianakiseleva/autogen/raw/ddabd4f0e7c13a50e33cf8462e79358666371477/test/test_files/agenteval-in-out/prealgebra.zip\n",
"Resolving github.com (github.com)... 140.82.116.4\n",
"Connecting to github.com (github.com)|140.82.116.4|:443... connected.\n",
"HTTP request sent, awaiting response... "
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"302 Found\n",
"--2024-04-29 16:40:49-- https://github.com/julianakiseleva/autogen/raw/ddabd4f0e7c13a50e33cf8462e79358666371477/test/test_files/agenteval-in-out/prealgebra.zip\n",
"Resolving github.com (github.com)... 140.82.116.3\n",
"Connecting to github.com (github.com)|140.82.116.3|:443... connected.\n",
"HTTP request sent, awaiting response... 302 Found\n",
"Location: https://raw.githubusercontent.com/julianakiseleva/autogen/ddabd4f0e7c13a50e33cf8462e79358666371477/test/test_files/agenteval-in-out/prealgebra.zip [following]\n",
"--2024-04-17 22:07:58-- https://raw.githubusercontent.com/julianakiseleva/autogen/ddabd4f0e7c13a50e33cf8462e79358666371477/test/test_files/agenteval-in-out/prealgebra.zip\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.111.133, 185.199.110.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
"--2024-04-29 16:40:49-- https://raw.githubusercontent.com/julianakiseleva/autogen/ddabd4f0e7c13a50e33cf8462e79358666371477/test/test_files/agenteval-in-out/prealgebra.zip\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.111.133, 185.199.108.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 28567 (28K) [application/zip]\n",
"Saving to: ‘prealgebra.zip’\n",
"\n",
"prealgebra.zip 100%[===================>] 27.90K --.-KB/s in 0.002s \n",
"prealgebra.zip 100%[===================>] 27.90K --.-KB/s in 0s \n",
"\n",
"2024-04-17 22:07:58 (11.3 MB/s) - ‘prealgebra.zip’ saved [28567/28567]\n",
"2024-04-29 16:40:49 (54.6 MB/s) - ‘prealgebra.zip’ saved [28567/28567]\n",
"\n",
"Archive: prealgebra.zip\n",
"warning: skipped \"../\" path component(s) in ../prealgebra/\n",
Expand Down Expand Up @@ -621,7 +592,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
Expand Down Expand Up @@ -801,13 +772,7 @@
"\n",
"--------------------------------------------------------------------------------\n",
"\u001b[31m\n",
">>>>>>>> USING AUTO REPLY...\u001b[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
">>>>>>>> USING AUTO REPLY...\u001b[0m\n",
"\u001b[33mquantifier\u001b[0m (to quantifier_user):\n",
"\n",
"{\n",
Expand Down Expand Up @@ -2558,7 +2523,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
Expand Down Expand Up @@ -2647,7 +2612,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
Expand Down

0 comments on commit be35fbc

Please sign in to comment.