From 5723e681b567d4aabe34989cf2eaa94715a8837b Mon Sep 17 00:00:00 2001 From: Yiran Wu <32823396+kevin666aa@users.noreply.github.com> Date: Sun, 9 Jun 2024 23:56:46 -0700 Subject: [PATCH 1/4] update --- .devcontainer/Dockerfile | 64 ++++++++++---------- autogen/oai/client.py | 29 ++++++++- notebook/agentchat_cost_token_tracking.ipynb | 53 +++++++++++++++- test/oai/test_client.py | 12 ++++ 4 files changed, 121 insertions(+), 37 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8a75a1487ff..755531953f4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,32 +1,32 @@ -#------------------------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE file in the project root for license information. -#------------------------------------------------------------------------------------------------------------- - -FROM mcr.microsoft.com/vscode/devcontainers/python:3.10 - -# -# Update the OS and maybe install packages -# -ENV DEBIAN_FRONTEND=noninteractive - -# add git lhs to apt -RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash - -RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get -y install --no-install-recommends build-essential npm git-lfs \ - && apt-get autoremove -y \ - && apt-get clean -y \ - && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \ - && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-${arch}.deb \ - && dpkg -i quarto-1.5.23-linux-${arch}.deb \ - && rm -rf /var/lib/apt/lists/* quarto-1.5.23-linux-${arch}.deb -ENV DEBIAN_FRONTEND=dialog - -# For docs -RUN npm install --global yarn -RUN pip install --upgrade pip -RUN pip install pydoc-markdown -RUN pip install pyyaml -RUN pip install colored +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE file in the project root for license information. +#------------------------------------------------------------------------------------------------------------- + +FROM mcr.microsoft.com/vscode/devcontainers/python:3.10 + +# +# Update the OS and maybe install packages +# +ENV DEBIAN_FRONTEND=noninteractive + +# add git lhs to apt +RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash + +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get -y install --no-install-recommends build-essential npm git-lfs \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \ + && wget https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-${arch}.deb \ + && dpkg -i quarto-1.5.23-linux-${arch}.deb \ + && rm -rf /var/lib/apt/lists/* quarto-1.5.23-linux-${arch}.deb +ENV DEBIAN_FRONTEND=dialog + +# For docs +RUN npm install --global yarn +RUN pip install --upgrade pip +RUN pip install pydoc-markdown +RUN pip install pyyaml +RUN pip install colored diff --git a/autogen/oai/client.py b/autogen/oai/client.py index 4c1da7a3931..b3869c7925c 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -291,7 +291,9 @@ def cost(self, response: Union[ChatCompletion, Completion]) -> float: model = response.model if model not in OAI_PRICE1K: # TODO: add logging to warn that the model is not found - logger.debug(f"Model {model} is not found. The cost will be 0.", exc_info=True) + logger.warning( + f'Model {model} is not found. The cost will be 0. In your config_list, add field {{"price" : [prompt_price_per_1k, completion_token_price_per_1k]}} for customized pricing.' + ) return 0 n_input_tokens = response.usage.prompt_tokens if response.usage is not None else 0 # type: ignore [union-attr] @@ -328,6 +330,7 @@ class OpenAIWrapper: "api_version", "api_type", "tags", + "price", } openai_kwargs = set(inspect.getfullargspec(OpenAI.__init__).kwonlyargs) @@ -592,6 +595,14 @@ def yes_or_no_filter(context, response): filter_func = extra_kwargs.get("filter_func") context = extra_kwargs.get("context") agent = extra_kwargs.get("agent") + price = extra_kwargs.get("price", None) + if isinstance(price, list): + price = tuple(price) + elif isinstance(price, float) or isinstance(price, int): + logger.warning( + "Input price is a float/int. Using the same price for prompt and completion tokens. Use a list/tuple if prompt and completion token prices are different." + ) + price = (price, price) total_usage = None actual_usage = None @@ -678,7 +689,10 @@ def yes_or_no_filter(context, response): raise else: # add cost calculation before caching no matter filter is passed or not - response.cost = client.cost(response) + if price is not None: + response.cost = self._cost_with_customized_price(response, price) + else: + response.cost = client.cost(response) actual_usage = client.get_usage(response) total_usage = actual_usage.copy() if actual_usage is not None else total_usage self._update_usage(actual_usage=actual_usage, total_usage=total_usage) @@ -712,6 +726,17 @@ def yes_or_no_filter(context, response): continue # filter is not passed; try the next config raise RuntimeError("Should not reach here.") + @staticmethod + def _cost_with_customized_price( + response: ModelClient.ModelClientResponseProtocol, price_1k: Tuple[float, float] + ) -> None: + """If a customized cost is passed, overwrite the cost in the response.""" + n_input_tokens = response.usage.prompt_tokens if response.usage is not None else 0 # type: ignore [union-attr] + n_output_tokens = response.usage.completion_tokens if response.usage is not None else 0 # type: ignore [union-attr] + if n_output_tokens is None: + n_output_tokens = 0 + return n_input_tokens * price_1k[0] + n_output_tokens * price_1k[1] + @staticmethod def _update_dict_from_chunk(chunk: BaseModel, d: Dict[str, Any], field: str) -> int: """Update the dict from the chunk. diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index 7feb7a908f4..aa520407190 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -31,6 +31,20 @@ "\n", "To gather usage data for a list of agents, we provide an utility function `autogen.gather_usage_summary(agents)` where you pass in a list of agents and gather the usage summary.\n", "\n", + "## 3. Custom token price for up-to-date cost estimation\n", + "AutoGen tries to keep the token prices up-to-date. However, you can pass in a `price` field in `config_list` if the token price is not listed or up-to-date. Please creating an issue or pull request to help us keep the token prices up-to-date!\n", + "\n", + "Note: in json files, the price should be a list of two floats.\n", + "\n", + "Example Usage:\n", + "```python\n", + "{\n", + " \"model\": \"gpt-3.5-turbo-xxxx\",\n", + " \"api_key\": \"YOUR_API_KEY\",\n", + " \"price\": [0.0005, 0.0015]\n", + "}\n", + "```\n", + "\n", "## Caution when using Azure OpenAI!\n", "If you are using azure OpenAI, the model returned from completion doesn't have the version information. The returned model is either 'gpt-35-turbo' or 'gpt-4'. From there, we are calculating the cost based on gpt-3.5-turbo-0125: (0.0005, 0.0015) per 1k prompt and completion tokens and gpt-4-0613: (0.03, 0.06). This means the cost can be wrong if you are using a different version from azure OpenAI.\n", "\n", @@ -55,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -65,7 +79,7 @@ "config_list = autogen.config_list_from_json(\n", " \"OAI_CONFIG_LIST\",\n", " filter_dict={\n", - " \"tags\": [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\"], # comment out to get all\n", + " \"model\": [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\"], # comment out to get all\n", " },\n", ")" ] @@ -127,6 +141,39 @@ "print(response.cost)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OpenAIWrapper with custom token price" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Price: 109\n" + ] + } + ], + "source": [ + "# Adding price to the config_list\n", + "for i in range(len(config_list)):\n", + " config_list[i][\"price\"] = [1, 1] # Note: This price is just for demonstration purposes. Please replace it with the actual price of the model.\n", + "\n", + "client = OpenAIWrapper(config_list=config_list)\n", + "messages = [\n", + " {\"role\": \"user\", \"content\": \"Can you give me 3 useful tips on learning Python? Keep it simple and short.\"},\n", + "]\n", + "response = client.create(messages=messages, cache_seed=None)\n", + "print(\"Price:\", response.cost)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -504,7 +551,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.9.19" } }, "nbformat": 4, diff --git a/test/oai/test_client.py b/test/oai/test_client.py index 1cbb4377772..b4b663e1882 100755 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -130,6 +130,18 @@ def test_cost(cache_seed): print(response.cost) +@pytest.mark.skipif(skip, reason="openai>=1 not installed") +def test_customized_cost(): + config_list = config_list_from_json( + env_or_file=OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"tags": ["gpt-3.5-turbo-instruct"]} + ) + config_list = config_list[0] + config_list.update({"price": [1, 1]}) + client = OpenAIWrapper(config_list=config_list, cache_seed=None) + response = client.create(prompt="1+3=") + assert response.cost >= 12, "Due to customized pricing, cost should be greater than 12" + + @pytest.mark.skipif(skip, reason="openai>=1 not installed") def test_usage_summary(): config_list = config_list_from_json( From e199c058f0e747c77c5a873e1b8529fca618f1b4 Mon Sep 17 00:00:00 2001 From: Yiran Wu <32823396+kevin666aa@users.noreply.github.com> Date: Mon, 10 Jun 2024 00:22:05 -0700 Subject: [PATCH 2/4] update --- notebook/agentchat_cost_token_tracking.ipynb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index aa520407190..99dacc744c7 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -164,7 +164,10 @@ "source": [ "# Adding price to the config_list\n", "for i in range(len(config_list)):\n", - " config_list[i][\"price\"] = [1, 1] # Note: This price is just for demonstration purposes. Please replace it with the actual price of the model.\n", + " config_list[i][\"price\"] = [\n", + " 1,\n", + " 1,\n", + " ] # Note: This price is just for demonstration purposes. Please replace it with the actual price of the model.\n", "\n", "client = OpenAIWrapper(config_list=config_list)\n", "messages = [\n", From f7d97c0a81878b97a2e0660c5b8da25af1a0739c Mon Sep 17 00:00:00 2001 From: Davor Runje Date: Tue, 11 Jun 2024 14:08:05 +0000 Subject: [PATCH 3/4] TODO comment removed --- autogen/oai/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogen/oai/client.py b/autogen/oai/client.py index b3869c7925c..5cfda44b710 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -290,7 +290,7 @@ def cost(self, response: Union[ChatCompletion, Completion]) -> float: """Calculate the cost of the response.""" model = response.model if model not in OAI_PRICE1K: - # TODO: add logging to warn that the model is not found + # log warning that the model is not found logger.warning( f'Model {model} is not found. The cost will be 0. In your config_list, add field {{"price" : [prompt_price_per_1k, completion_token_price_per_1k]}} for customized pricing.' ) From 069dd1cdb82c7d78750edd526a9f4ad836cb1bf8 Mon Sep 17 00:00:00 2001 From: kevin666aa Date: Thu, 13 Jun 2024 11:16:28 -0400 Subject: [PATCH 4/4] update --- test/oai/test_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/oai/test_client.py b/test/oai/test_client.py index b4b663e1882..d4878d640e5 100755 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -135,11 +135,11 @@ def test_customized_cost(): config_list = config_list_from_json( env_or_file=OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"tags": ["gpt-3.5-turbo-instruct"]} ) - config_list = config_list[0] - config_list.update({"price": [1, 1]}) + for config in config_list: + config.update({"price": [1, 1]}) client = OpenAIWrapper(config_list=config_list, cache_seed=None) response = client.create(prompt="1+3=") - assert response.cost >= 12, "Due to customized pricing, cost should be greater than 12" + assert response.cost >= 4, "Due to customized pricing, cost should be greater than 4" @pytest.mark.skipif(skip, reason="openai>=1 not installed")