From 69ac4c84ae82bd34ed0ad7b3e119be13e087f570 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 15:15:15 -0400 Subject: [PATCH 01/13] Add support models via openai sdk --- log10/_httpx_utils.py | 85 +++++++++++++++++++++++++++++++++---------- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index ba1c1f31..c4931561 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -6,6 +6,7 @@ import traceback import uuid from datetime import datetime, timezone +from enum import Enum import httpx from httpx import Request, Response @@ -29,6 +30,24 @@ httpx_async_client = httpx.AsyncClient(timeout=timeout) +class LLM_PROVIDER(Enum): + ANTHROPIC = "Anthropic" + OPENAI = "OpenAI" + UNKNOWN = "Unknown" + + +PROVIDER_PATHS = { + LLM_PROVIDER.ANTHROPIC: ["/v1/messages", "/v1/complete"], + LLM_PROVIDER.OPENAI: ["v1/chat/completions"], +} + +USER_AGENT_NAME_TO_PROVIDER = { + "AsyncOpenAI": LLM_PROVIDER.OPENAI, + "AsyncAnthropic": LLM_PROVIDER.ANTHROPIC, + "Anthropic": LLM_PROVIDER.ANTHROPIC, +} + + def _get_time_diff(created_at): time = datetime.fromisoformat(created_at) now = datetime.now(timezone.utc) @@ -225,14 +244,28 @@ def format_anthropic_request(request_content) -> str: return json.dumps(request_content) +def _get_llm_provider(request: Request) -> LLM_PROVIDER: + user_agent = request.headers.get("user-agent", "") + class_name = user_agent.split("/")[0] + + if class_name in ["AsyncAnthropic", "Anthropic"]: + return LLM_PROVIDER.ANTHROPIC + elif class_name in ["AsyncOpenAI"]: + return LLM_PROVIDER.OPENAI + else: + return LLM_PROVIDER.UNKNOWN + + def _init_log_row(request: Request): start_time = time.time() request.started = start_time orig_module = "" orig_qualname = "" request_content_decode = request.content.decode("utf-8") - host = request.headers.get("host") - if "openai" in host: + llm_provider = _get_llm_provider(request) + + # host = request.headers.get("host") + if llm_provider == LLM_PROVIDER.OPENAI: if "chat" in str(request.url): kind = "chat" orig_module = "openai.api_resources.chat_completion" @@ -241,7 +274,7 @@ def _init_log_row(request: Request): kind = "completion" orig_module = "openai.api_resources.completion" orig_qualname = "Completion.create" - elif "anthropic" in host: + elif llm_provider == LLM_PROVIDER.ANTHROPIC: kind = "chat" url_path = request.url content_type = request.headers.get("content-type") @@ -259,10 +292,15 @@ def _init_log_row(request: Request): orig_qualname = "Completions.create" request_content_decode = format_anthropic_request(request_content) + # elif "mistral" in host: + # kind = "chat" + # orig_module = "mistral.api_resources.chat" ## generated not correct + # orig_qualname = "Chat" ## generated not correct else: logger.debug("Currently logging is only available for async openai and anthropic.") return + log_row = { "status": "started", "kind": kind, @@ -278,15 +316,21 @@ def _init_log_row(request: Request): def get_completion_id(request: Request): - host = request.headers.get("host") - if "anthropic" in host: - paths = ["/v1/messages", "/v1/complete"] - if not any(path in str(request.url) for path in paths): - logger.debug("Currently logging is only available for anthropic v1/messages and v1/complete.") - return + # request_user_agent = request.headers.get("user-agent") + # logger.info(f"Request_user_agent: {request_user_agent}") + # allowed_class_names = ["AsyncOpenAI", "AsyncAnthropic", "Anthropic"] + # request_class_name = request_user_agent.split("/")[0] + + llm_provider = _get_llm_provider(request) + if llm_provider is LLM_PROVIDER.UNKNOWN: + logger.debug("Currently logging is only available for async openai and anthropic.") + return - if "openai" in host and "v1/chat/completions" not in str(request.url): - logger.debug("Currently logging is only available for openai v1/chat/completions.") + # Check if the request URL matches any of the allowed paths for the class name + if not any(path in str(request.url) for path in PROVIDER_PATHS.get(llm_provider, [])): + logger.debug( + f'Currently logging is only available for {llm_provider} {', '.join(PROVIDER_PATHS[llm_provider])}.' + ) return completion_id = str(uuid.uuid4()) @@ -361,6 +405,10 @@ def log_request(self, request: httpx.Request): logger.debug("LOG10: sending sync request") self.log_row = _init_log_row(request) + if not self.log_row: + logger.debug("LOG10: log row is not initialized. Skipping") + return + _try_post_request(url=f"{base_url}/api/completions/{completion_id}", payload=self.log_row) @@ -396,6 +444,7 @@ async def log_request(self, request: httpx.Request): class _LogResponse(Response): def __init__(self, *args, **kwargs): self.log_row = kwargs.pop("log_row") + self.llm_provider = _get_llm_provider(kwargs.get("request")) super().__init__(*args, **kwargs) def patch_streaming_log(self, duration: int, full_response: str): @@ -452,11 +501,10 @@ async def aiter_bytes(self, *args, **kwargs): ) yield chunk - def is_response_end_reached(self, text: str): - host = self.request.headers.get("host") - if "anthropic" in host: + def is_response_end_reached(self, text: str) -> bool: + if self.llm_provider == LLM_PROVIDER.ANTHROPIC: return self.is_anthropic_response_end_reached(text) - elif "openai" in host: + elif self.llm_provider == LLM_PROVIDER.OPENAI: return self.is_openai_response_end_reached(text) else: logger.debug("Currently logging is only available for async openai and anthropic.") @@ -631,11 +679,10 @@ def parse_openai_responses(self, responses: list[str]): return response_json def parse_response_data(self, responses: list[str]): - host = self.request.headers.get("host") - if "openai" in host: - return self.parse_openai_responses(responses) - elif "anthropic" in host: + if self.llm_provider == LLM_PROVIDER.ANTHROPIC: return self.parse_anthropic_responses(responses) + elif self.llm_provider == LLM_PROVIDER.OPENAI: + return self.parse_openai_responses(responses) else: logger.debug("Currently logging is only available for async openai and anthropic.") return None From 0d4f0b5a62e19cc90d10026b7f0c393e2007c3c0 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 17:50:30 -0400 Subject: [PATCH 02/13] Support perplexity model via openai sdk --- log10/_httpx_utils.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index c4931561..77c53cc0 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -38,7 +38,7 @@ class LLM_PROVIDER(Enum): PROVIDER_PATHS = { LLM_PROVIDER.ANTHROPIC: ["/v1/messages", "/v1/complete"], - LLM_PROVIDER.OPENAI: ["v1/chat/completions"], + LLM_PROVIDER.OPENAI: ["v1/chat/completions", "chat/completions"], } USER_AGENT_NAME_TO_PROVIDER = { @@ -264,7 +264,6 @@ def _init_log_row(request: Request): request_content_decode = request.content.decode("utf-8") llm_provider = _get_llm_provider(request) - # host = request.headers.get("host") if llm_provider == LLM_PROVIDER.OPENAI: if "chat" in str(request.url): kind = "chat" @@ -292,11 +291,6 @@ def _init_log_row(request: Request): orig_qualname = "Completions.create" request_content_decode = format_anthropic_request(request_content) - # elif "mistral" in host: - # kind = "chat" - # orig_module = "mistral.api_resources.chat" ## generated not correct - # orig_qualname = "Chat" ## generated not correct - else: logger.debug("Currently logging is only available for async openai and anthropic.") return @@ -316,11 +310,6 @@ def _init_log_row(request: Request): def get_completion_id(request: Request): - # request_user_agent = request.headers.get("user-agent") - # logger.info(f"Request_user_agent: {request_user_agent}") - # allowed_class_names = ["AsyncOpenAI", "AsyncAnthropic", "Anthropic"] - # request_class_name = request_user_agent.split("/")[0] - llm_provider = _get_llm_provider(request) if llm_provider is LLM_PROVIDER.UNKNOWN: logger.debug("Currently logging is only available for async openai and anthropic.") @@ -436,6 +425,10 @@ async def log_request(self, request: httpx.Request): logger.debug("LOG10: sending async request") self.log_row = _init_log_row(request) + if not self.log_row: + logger.debug("LOG10: log row is not initialized. Skipping") + return + asyncio.create_task( _try_post_request_async(url=f"{base_url}/api/completions/{completion_id}", payload=self.log_row) ) @@ -445,6 +438,7 @@ class _LogResponse(Response): def __init__(self, *args, **kwargs): self.log_row = kwargs.pop("log_row") self.llm_provider = _get_llm_provider(kwargs.get("request")) + self.host_header = kwargs.get("request").headers.get("host") super().__init__(*args, **kwargs) def patch_streaming_log(self, duration: int, full_response: str): @@ -459,7 +453,10 @@ def patch_streaming_log(self, duration: int, full_response: str): for frame in current_stack_frame ] - responses = full_response.split("\n\n") + separator = ( + "\r\n\r\n" if self.llm_provider == LLM_PROVIDER.OPENAI and "perplexity" in self.host_header else "\n\n" + ) + responses = full_response.split(separator) response_json = self.parse_response_data(responses) self.log_row["response"] = json.dumps(response_json) @@ -505,7 +502,10 @@ def is_response_end_reached(self, text: str) -> bool: if self.llm_provider == LLM_PROVIDER.ANTHROPIC: return self.is_anthropic_response_end_reached(text) elif self.llm_provider == LLM_PROVIDER.OPENAI: - return self.is_openai_response_end_reached(text) + if "perplexity" in self.host_header: + return self.is_perplexity_response_end_reached(text) + else: + return self.is_openai_response_end_reached(text) else: logger.debug("Currently logging is only available for async openai and anthropic.") return False @@ -513,6 +513,13 @@ def is_response_end_reached(self, text: str) -> bool: def is_anthropic_response_end_reached(self, text: str): return "event: message_stop" in text + def is_perplexity_response_end_reached(self, text: str): + json_strings = text.split("data: ")[1:] + # Parse the last JSON string + last_json_str = json_strings[-1].strip() + last_object = json.loads(last_json_str) + return last_object.get("choices", [{}])[0].get("finish_reason", "") == "stop" + def is_openai_response_end_reached(self, text: str): return "data: [DONE]" in text @@ -612,7 +619,8 @@ def parse_openai_responses(self, responses: list[str]): finish_reason = "" for r in responses: - if self.is_openai_response_end_reached(r): + # For perplexity, the last item in the responses is empty + if self.is_openai_response_end_reached(r) or not r: break # loading the substring of response text after 'data: '. From 4f3c06bc29ba57240d5fd140e3dd0c29c8d34f28 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 17:57:02 -0400 Subject: [PATCH 03/13] Fix quotes around f-string --- log10/_httpx_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index 77c53cc0..949ff278 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -318,7 +318,7 @@ def get_completion_id(request: Request): # Check if the request URL matches any of the allowed paths for the class name if not any(path in str(request.url) for path in PROVIDER_PATHS.get(llm_provider, [])): logger.debug( - f'Currently logging is only available for {llm_provider} {', '.join(PROVIDER_PATHS[llm_provider])}.' + f"Currently logging is only available for {llm_provider} {', '.join(PROVIDER_PATHS[llm_provider])}." ) return From 3ceff62e81d3a16cdf8e0ea6c09cbc9de67d2325 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 21:43:10 -0400 Subject: [PATCH 04/13] Update tests to run against openai perplexity compability in openai and magentic --- .github/workflows/test.yml | 2 + log10/_httpx_utils.py | 43 ++++--- tests/conftest.py | 17 ++- tests/pytest.ini | 1 + tests/test_magentic.py | 4 + tests/test_openai_perplexity_compability.py | 135 ++++++++++++++++++++ 6 files changed, 177 insertions(+), 25 deletions(-) create mode 100644 tests/test_openai_perplexity_compability.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 036f6062..428f46c7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -118,6 +118,7 @@ jobs: echo "All variables are empty" poetry run pytest -vv tests/ --ignore=tests/test_cli.py poetry run pytest --llm_provider=anthropic -vv tests/test_magentic.py + poetry run pytest --llm_provider=litellm --openai_compability_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat fi - name: Run scheduled llm tests @@ -126,3 +127,4 @@ jobs: echo "This is a schedule event" poetry run pytest -vv tests/ --ignore=tests/test_cli.py poetry run pytest --openai_model=gpt-4o -m chat -vv tests/test_openai.py + poetry run pytest --llm_provider=litellm --openai_compability_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index 949ff278..b780f5a5 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -30,21 +30,22 @@ httpx_async_client = httpx.AsyncClient(timeout=timeout) -class LLM_PROVIDER(Enum): +class LLM_CLIENTS(Enum): ANTHROPIC = "Anthropic" OPENAI = "OpenAI" UNKNOWN = "Unknown" PROVIDER_PATHS = { - LLM_PROVIDER.ANTHROPIC: ["/v1/messages", "/v1/complete"], - LLM_PROVIDER.OPENAI: ["v1/chat/completions", "chat/completions"], + LLM_CLIENTS.ANTHROPIC: ["/v1/messages", "/v1/complete"], + LLM_CLIENTS.OPENAI: ["v1/chat/completions", "chat/completions"], } USER_AGENT_NAME_TO_PROVIDER = { - "AsyncOpenAI": LLM_PROVIDER.OPENAI, - "AsyncAnthropic": LLM_PROVIDER.ANTHROPIC, - "Anthropic": LLM_PROVIDER.ANTHROPIC, + "AsyncOpenAI": LLM_CLIENTS.OPENAI, + "AsyncAnthropic": LLM_CLIENTS.ANTHROPIC, + "Anthropic": LLM_CLIENTS.ANTHROPIC, + "OpenAI": LLM_CLIENTS.OPENAI, } @@ -244,16 +245,16 @@ def format_anthropic_request(request_content) -> str: return json.dumps(request_content) -def _get_llm_provider(request: Request) -> LLM_PROVIDER: +def _get_llm_provider(request: Request) -> LLM_CLIENTS: user_agent = request.headers.get("user-agent", "") class_name = user_agent.split("/")[0] if class_name in ["AsyncAnthropic", "Anthropic"]: - return LLM_PROVIDER.ANTHROPIC + return LLM_CLIENTS.ANTHROPIC elif class_name in ["AsyncOpenAI"]: - return LLM_PROVIDER.OPENAI + return LLM_CLIENTS.OPENAI else: - return LLM_PROVIDER.UNKNOWN + return LLM_CLIENTS.UNKNOWN def _init_log_row(request: Request): @@ -264,7 +265,7 @@ def _init_log_row(request: Request): request_content_decode = request.content.decode("utf-8") llm_provider = _get_llm_provider(request) - if llm_provider == LLM_PROVIDER.OPENAI: + if llm_provider == LLM_CLIENTS.OPENAI: if "chat" in str(request.url): kind = "chat" orig_module = "openai.api_resources.chat_completion" @@ -273,7 +274,7 @@ def _init_log_row(request: Request): kind = "completion" orig_module = "openai.api_resources.completion" orig_qualname = "Completion.create" - elif llm_provider == LLM_PROVIDER.ANTHROPIC: + elif llm_provider == LLM_CLIENTS.ANTHROPIC: kind = "chat" url_path = request.url content_type = request.headers.get("content-type") @@ -311,7 +312,7 @@ def _init_log_row(request: Request): def get_completion_id(request: Request): llm_provider = _get_llm_provider(request) - if llm_provider is LLM_PROVIDER.UNKNOWN: + if llm_provider is LLM_CLIENTS.UNKNOWN: logger.debug("Currently logging is only available for async openai and anthropic.") return @@ -454,7 +455,7 @@ def patch_streaming_log(self, duration: int, full_response: str): ] separator = ( - "\r\n\r\n" if self.llm_provider == LLM_PROVIDER.OPENAI and "perplexity" in self.host_header else "\n\n" + "\r\n\r\n" if self.llm_provider == LLM_CLIENTS.OPENAI and "perplexity" in self.host_header else "\n\n" ) responses = full_response.split(separator) response_json = self.parse_response_data(responses) @@ -499,9 +500,9 @@ async def aiter_bytes(self, *args, **kwargs): yield chunk def is_response_end_reached(self, text: str) -> bool: - if self.llm_provider == LLM_PROVIDER.ANTHROPIC: + if self.llm_provider == LLM_CLIENTS.ANTHROPIC: return self.is_anthropic_response_end_reached(text) - elif self.llm_provider == LLM_PROVIDER.OPENAI: + elif self.llm_provider == LLM_CLIENTS.OPENAI: if "perplexity" in self.host_header: return self.is_perplexity_response_end_reached(text) else: @@ -521,7 +522,8 @@ def is_perplexity_response_end_reached(self, text: str): return last_object.get("choices", [{}])[0].get("finish_reason", "") == "stop" def is_openai_response_end_reached(self, text: str): - return "data: [DONE]" in text + # For perplexity, the last item in the responses is empty + return "data: [DONE]" in text or not text def parse_anthropic_responses(self, responses: list[str]): message_id = "" @@ -619,8 +621,7 @@ def parse_openai_responses(self, responses: list[str]): finish_reason = "" for r in responses: - # For perplexity, the last item in the responses is empty - if self.is_openai_response_end_reached(r) or not r: + if self.is_openai_response_end_reached(r): break # loading the substring of response text after 'data: '. @@ -687,9 +688,9 @@ def parse_openai_responses(self, responses: list[str]): return response_json def parse_response_data(self, responses: list[str]): - if self.llm_provider == LLM_PROVIDER.ANTHROPIC: + if self.llm_provider == LLM_CLIENTS.ANTHROPIC: return self.parse_anthropic_responses(responses) - elif self.llm_provider == LLM_PROVIDER.OPENAI: + elif self.llm_provider == LLM_CLIENTS.OPENAI: return self.parse_openai_responses(responses) else: logger.debug("Currently logging is only available for async openai and anthropic.") diff --git a/tests/conftest.py b/tests/conftest.py index 5e266c08..809a9b4b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,10 @@ def pytest_addoption(parser): parser.addoption("--llm_provider", action="store", help="Model provider name for Magentic tests") + parser.addoption( + "--openai_compability_model", action="store", help="Model name for client compability model in Magentic tests" + ) + @pytest.fixture def openai_model(request): @@ -66,12 +70,17 @@ def llm_provider(request): @pytest.fixture def magentic_models(request): llm_provider = request.config.getoption("--llm_provider") - model_config_name = f"--{llm_provider}_model" - vision_model_config_name = llm_provider == "openai" and f"--{llm_provider}_vision_model" or model_config_name + model_configs_to_providers = { + "openai": ["openai_model", "openai_vision_model"], + "anthropic": ["anthropic_model", "anthropic_model"], + "litellm": ["openai_compability_model", "openai_compability_model"], + } + + model_configs = model_configs_to_providers[llm_provider] return { - "chat_model": request.config.getoption(model_config_name), - "vision_model": request.config.getoption(vision_model_config_name), + "chat_model": request.config.getoption(model_configs[0]), + "vision_model": request.config.getoption(model_configs[1]), } diff --git a/tests/pytest.ini b/tests/pytest.ini index 65219fb1..f655cc15 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -7,6 +7,7 @@ addopts = --google_model=gemini-1.5-pro-latest --mistralai_model=mistral-tiny --lamini_model=meta-llama/Llama-2-7b-chat-hf + --openai_compability_model=perplexity/llama-3.1-sonar-small-128k-chat --llm_provider=openai markers = diff --git a/tests/test_magentic.py b/tests/test_magentic.py index ee99a888..d930c4bd 100644 --- a/tests/test_magentic.py +++ b/tests/test_magentic.py @@ -15,6 +15,7 @@ prompt, ) from magentic.chat_model.anthropic_chat_model import AnthropicChatModel +from magentic.chat_model.litellm_chat_model import LitellmChatModel from magentic.vision import UserImageMessage from pydantic import BaseModel @@ -27,6 +28,7 @@ def _get_model_obj(llm_provider, model, params): provider_map = { "openai": (OpenaiChatModel, (log10, {"module": openai})), "anthropic": (AnthropicChatModel, (log10, {"module": anthropic})), + "litellm": (LitellmChatModel, (log10, {"module": openai})), } if llm_provider not in provider_map: @@ -102,6 +104,7 @@ def configure_oven(food: str) -> FunctionCall[str]: # ruff: ignore _LogAssertion(completion_id=session.last_completion_id(), function_args=function_args).assert_tool_calls_response() +@pytest.mark.chat @pytest.mark.async_client @pytest.mark.stream @pytest.mark.asyncio(scope="module") @@ -146,6 +149,7 @@ async def plus_and_minus(a: int, b: int) -> AsyncParallelFunctionCall[int]: ... _LogAssertion(completion_id=session.last_completion_id(), function_args=function_args).assert_tool_calls_response() +@pytest.mark.chat @pytest.mark.async_client @pytest.mark.stream @pytest.mark.asyncio(scope="module") diff --git a/tests/test_openai_perplexity_compability.py b/tests/test_openai_perplexity_compability.py new file mode 100644 index 00000000..c78c5444 --- /dev/null +++ b/tests/test_openai_perplexity_compability.py @@ -0,0 +1,135 @@ +import os + +import openai +import pytest +from openai import NOT_GIVEN, AsyncOpenAI + +from log10._httpx_utils import finalize +from log10.load import log10 +from tests.utils import _LogAssertion + + +log10(openai) + +model_name = "llama-3.1-sonar-small-128k-chat" + +if "PERPLEXITYAI_API_KEY" not in os.environ: + raise ValueError("Please set the PERPLEXITYAI_API_KEY environment variable.") + +compability_config = { + "base_url": "https://api.perplexity.ai", + "api_key": os.environ.get("PERPLEXITYAI_API_KEY"), +} + + +@pytest.mark.chat +def test_chat(session): + client = openai.OpenAI(**compability_config) + completion = client.chat.completions.create( + model=model_name, + messages=[ + { + "role": "system", + "content": "You will be provided with statements, and your task is to convert them to standard English.", + }, + { + "role": "user", + "content": "He no went to the market.", + }, + ], + ) + + content = completion.choices[0].message.content + assert isinstance(content, str) + assert session.last_completion_url() is not None, "No completion URL found." + _LogAssertion(completion_id=session.last_completion_id(), message_content=content).assert_chat_response() + + +@pytest.mark.chat +def test_chat_not_given(session): + client = openai.OpenAI(**compability_config) + completion = client.chat.completions.create( + model=model_name, + messages=[ + { + "role": "user", + "content": "tell a short joke.", + }, + ], + tools=NOT_GIVEN, + tool_choice=NOT_GIVEN, + ) + + content = completion.choices[0].message.content + assert isinstance(content, str) + assert session.last_completion_url() is not None, "No completion URL found." + _LogAssertion(completion_id=session.last_completion_id(), message_content=content).assert_chat_response() + + +@pytest.mark.chat +@pytest.mark.async_client +@pytest.mark.asyncio(scope="module") +async def test_chat_async(session): + client = AsyncOpenAI(**compability_config) + completion = await client.chat.completions.create( + model=model_name, + messages=[{"role": "user", "content": "Say this is a test"}], + ) + + content = completion.choices[0].message.content + assert isinstance(content, str) + await finalize() + _LogAssertion(completion_id=session.last_completion_id(), message_content=content).assert_chat_response() + + +@pytest.mark.chat +@pytest.mark.async_client +@pytest.mark.asyncio(scope="module") +async def test_perplexity_chat_async(session): + client = AsyncOpenAI(**compability_config) + completion = await client.chat.completions.create( + model=model_name, + messages=[{"role": "user", "content": "Say this is a test"}], + ) + + content = completion.choices[0].message.content + assert isinstance(content, str) + await finalize() + _LogAssertion(completion_id=session.last_completion_id(), message_content=content).assert_chat_response() + + +@pytest.mark.chat +@pytest.mark.stream +def test_chat_stream(session): + client = openai.OpenAI(**compability_config) + response = client.chat.completions.create( + model=model_name, + messages=[{"role": "user", "content": "Count to 5"}], + temperature=0, + stream=True, + ) + + output = "" + for chunk in response: + output += chunk.choices[0].delta.content + + _LogAssertion(completion_id=session.last_completion_id(), message_content=output).assert_chat_response() + + +@pytest.mark.async_client +@pytest.mark.stream +@pytest.mark.asyncio(scope="module") +async def test_chat_async_stream(session): + client = AsyncOpenAI(**compability_config) + + output = "" + stream = await client.chat.completions.create( + model=model_name, + messages=[{"role": "user", "content": "Count to 8"}], + stream=True, + ) + async for chunk in stream: + output += chunk.choices[0].delta.content or "" + + await finalize() + _LogAssertion(completion_id=session.last_completion_id(), message_content=output).assert_chat_response() From 98ae5de80a2d6f1f6f5f8c2492690635faef4308 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 21:56:02 -0400 Subject: [PATCH 05/13] Add openai perplexity compability logging examples --- ...sync_chat_openai_perplexity_compability.py | 26 +++++++++++++++++++ ...ai_async_logging_perplexity_compability.py | 25 ++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 examples/logging/magentic_async_chat_openai_perplexity_compability.py create mode 100644 examples/logging/openai_async_logging_perplexity_compability.py diff --git a/examples/logging/magentic_async_chat_openai_perplexity_compability.py b/examples/logging/magentic_async_chat_openai_perplexity_compability.py new file mode 100644 index 00000000..f3ffcb58 --- /dev/null +++ b/examples/logging/magentic_async_chat_openai_perplexity_compability.py @@ -0,0 +1,26 @@ +import asyncio + +import openai +from magentic import UserMessage, chatprompt +from magentic.chat_model.litellm_chat_model import LitellmChatModel +from log10._httpx_utils import finalize +from log10.load import log10 + + +log10(openai) + + +async def main(topic: str) -> str: + @chatprompt( + UserMessage(f"Tell me a joke about {topic}"), + model=LitellmChatModel( + model="perplexity/llama-3.1-sonar-small-128k-chat" + ), + ) + async def tell_joke(topic: str) -> str: ... + + print(await tell_joke(topic)) + await finalize() + + +asyncio.run(main("cats")) diff --git a/examples/logging/openai_async_logging_perplexity_compability.py b/examples/logging/openai_async_logging_perplexity_compability.py new file mode 100644 index 00000000..3df1cf6e --- /dev/null +++ b/examples/logging/openai_async_logging_perplexity_compability.py @@ -0,0 +1,25 @@ +import asyncio +import os + +import openai +from openai import AsyncOpenAI + +from log10._httpx_utils import finalize +from log10.load import log10 + + +log10(openai) + +client = AsyncOpenAI(base_url="https://api.perplexity.ai", api_key=os.environ.get("PERPLEXITYAI_API_KEY")) + + +async def main(): + completion = await client.chat.completions.create( + model="llama-3.1-sonar-small-128k-chat", + messages=[{"role": "user", "content": "Say this is a test"}], + ) + print(completion.choices[0].message.content) + await finalize() + + +asyncio.run(main()) From 7e7255b6e3fd0b7a2f92f16f8bd3b36fe59c5d95 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 21:57:21 -0400 Subject: [PATCH 06/13] Format files --- .../magentic_async_chat_openai_perplexity_compability.py | 5 ++--- .../logging/openai_async_logging_perplexity_compability.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/logging/magentic_async_chat_openai_perplexity_compability.py b/examples/logging/magentic_async_chat_openai_perplexity_compability.py index f3ffcb58..2aaf5673 100644 --- a/examples/logging/magentic_async_chat_openai_perplexity_compability.py +++ b/examples/logging/magentic_async_chat_openai_perplexity_compability.py @@ -3,6 +3,7 @@ import openai from magentic import UserMessage, chatprompt from magentic.chat_model.litellm_chat_model import LitellmChatModel + from log10._httpx_utils import finalize from log10.load import log10 @@ -13,9 +14,7 @@ async def main(topic: str) -> str: @chatprompt( UserMessage(f"Tell me a joke about {topic}"), - model=LitellmChatModel( - model="perplexity/llama-3.1-sonar-small-128k-chat" - ), + model=LitellmChatModel(model="perplexity/llama-3.1-sonar-small-128k-chat"), ) async def tell_joke(topic: str) -> str: ... diff --git a/examples/logging/openai_async_logging_perplexity_compability.py b/examples/logging/openai_async_logging_perplexity_compability.py index 3df1cf6e..8964afe1 100644 --- a/examples/logging/openai_async_logging_perplexity_compability.py +++ b/examples/logging/openai_async_logging_perplexity_compability.py @@ -10,7 +10,7 @@ log10(openai) -client = AsyncOpenAI(base_url="https://api.perplexity.ai", api_key=os.environ.get("PERPLEXITYAI_API_KEY")) +client = AsyncOpenAI(base_url="https://api.perplexity.ai", api_key=os.environ.get("PERPLEXITYAI_API_KEY")) async def main(): From e9240066bff94ad4af8f67520e391cfa5d914663 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 21:58:22 -0400 Subject: [PATCH 07/13] Add perplexity api key env --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 428f46c7..c4b96c0a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,6 +50,7 @@ jobs: MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} LAMINI_API_KEY: ${{ secrets.LAMINI_API_KEY }} GOOGLE_API_KEY : ${{ secrets.GOOGLE_API_KEY }} + PERPLEXITYAI_API_KEY: ${{ secrets.PERPLEXITYAI_API_KEY }} steps: - uses: actions/checkout@v4 - name: Install poetry From 037f71e172b828737a51d201a1dbd43ce677d242 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 22:04:58 -0400 Subject: [PATCH 08/13] Upgrade litellm for magentic --- poetry.lock | 266 ++++++++++++++++++++++++++++++++++++++----------- pyproject.toml | 2 +- 2 files changed, 211 insertions(+), 57 deletions(-) diff --git a/poetry.lock b/poetry.lock index a9d05352..4d4e22df 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -1681,6 +1681,41 @@ files = [ {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, ] +[[package]] +name = "jsonschema" +version = "4.23.0" +description = "An implementation of JSON Schema validation for Python" +optional = true +python-versions = ">=3.8" +files = [ + {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, + {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +jsonschema-specifications = ">=2023.03.6" +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=24.6.0)"] + +[[package]] +name = "jsonschema-specifications" +version = "2023.12.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = true +python-versions = ">=3.8" +files = [ + {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, + {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, +] + +[package.dependencies] +referencing = ">=0.31.0" + [[package]] name = "lamini" version = "2.1.8" @@ -1847,13 +1882,13 @@ requests = ">=2,<3" [[package]] name = "litellm" -version = "1.35.32" +version = "1.44.2" description = "Library to easily interface with LLM API providers" optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" files = [ - {file = "litellm-1.35.32-py3-none-any.whl", hash = "sha256:4c05b90124420e64d4a59a4b94d6b53eb679859cd064e7e59bd152a29cda1cb1"}, - {file = "litellm-1.35.32.tar.gz", hash = "sha256:2ccc0f979c287404ec07c6b4b06254eba88f2ca1fca4cad12591ec89795db248"}, + {file = "litellm-1.44.2-py3-none-any.whl", hash = "sha256:7c94618f06c5990f8103b65a6752c7713da334524f9c0334346b8f63df5cdfb5"}, + {file = "litellm-1.44.2.tar.gz", hash = "sha256:b9290b1414caf6a955a5a493fb1b1dce95b2710a934e968c9ab04d0a3fba804c"}, ] [package.dependencies] @@ -1861,15 +1896,17 @@ aiohttp = "*" click = "*" importlib-metadata = ">=6.8.0" jinja2 = ">=3.1.2,<4.0.0" -openai = ">=1.0.0" +jsonschema = ">=4.22.0,<5.0.0" +openai = ">=1.40.0" +pydantic = ">=2.0.0,<3.0.0" python-dotenv = ">=0.2.0" requests = ">=2.31.0,<3.0.0" -tiktoken = ">=0.4.0" +tiktoken = ">=0.7.0" tokenizers = "*" [package.extras] -extra-proxy = ["azure-identity (>=1.15.0,<2.0.0)", "azure-keyvault-secrets (>=4.8.0,<5.0.0)", "google-cloud-kms (>=2.21.3,<3.0.0)", "prisma (==0.11.0)", "resend (>=0.8.0,<0.9.0)"] -proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=42.0.5,<43.0.0)", "fastapi (>=0.109.1,<0.110.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=21.2.0,<22.0.0)", "orjson (>=3.9.7,<4.0.0)", "python-multipart (>=0.0.9,<0.0.10)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] +extra-proxy = ["azure-identity (>=1.15.0,<2.0.0)", "azure-keyvault-secrets (>=4.8.0,<5.0.0)", "google-cloud-kms (>=2.21.3,<3.0.0)", "prisma (==0.11.0)", "pynacl (>=1.5.0,<2.0.0)", "resend (>=0.8.0,<0.9.0)"] +proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=42.0.5,<43.0.0)", "fastapi (>=0.111.0,<0.112.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=22.0.0,<23.0.0)", "orjson (>=3.9.7,<4.0.0)", "python-multipart (>=0.0.9,<0.0.10)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] [[package]] name = "logfire-api" @@ -2761,7 +2798,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2769,15 +2805,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2794,7 +2823,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2802,7 +2830,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2825,6 +2852,21 @@ prompt_toolkit = ">=2.0,<4.0" [package.extras] docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphinx-autodoc-typehints (>=1.11.1,<2.0.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "sphinx-rtd-theme (>=0.5.0,<0.6.0)"] +[[package]] +name = "referencing" +version = "0.35.1" +description = "JSON Referencing + Python" +optional = true +python-versions = ">=3.8" +files = [ + {file = "referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de"}, + {file = "referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + [[package]] name = "regex" version = "2024.4.28" @@ -2983,6 +3025,118 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "rpds-py" +version = "0.20.0" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = true +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.20.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2"}, + {file = "rpds_py-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf"}, + {file = "rpds_py-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce"}, + {file = "rpds_py-0.20.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94"}, + {file = "rpds_py-0.20.0-cp310-none-win32.whl", hash = "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee"}, + {file = "rpds_py-0.20.0-cp310-none-win_amd64.whl", hash = "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399"}, + {file = "rpds_py-0.20.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489"}, + {file = "rpds_py-0.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209"}, + {file = "rpds_py-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad"}, + {file = "rpds_py-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58"}, + {file = "rpds_py-0.20.0-cp311-none-win32.whl", hash = "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0"}, + {file = "rpds_py-0.20.0-cp311-none-win_amd64.whl", hash = "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c"}, + {file = "rpds_py-0.20.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6"}, + {file = "rpds_py-0.20.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4"}, + {file = "rpds_py-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940"}, + {file = "rpds_py-0.20.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174"}, + {file = "rpds_py-0.20.0-cp312-none-win32.whl", hash = "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139"}, + {file = "rpds_py-0.20.0-cp312-none-win_amd64.whl", hash = "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585"}, + {file = "rpds_py-0.20.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29"}, + {file = "rpds_py-0.20.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879"}, + {file = "rpds_py-0.20.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2"}, + {file = "rpds_py-0.20.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57"}, + {file = "rpds_py-0.20.0-cp313-none-win32.whl", hash = "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a"}, + {file = "rpds_py-0.20.0-cp313-none-win_amd64.whl", hash = "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2"}, + {file = "rpds_py-0.20.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24"}, + {file = "rpds_py-0.20.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751"}, + {file = "rpds_py-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253"}, + {file = "rpds_py-0.20.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a"}, + {file = "rpds_py-0.20.0-cp38-none-win32.whl", hash = "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5"}, + {file = "rpds_py-0.20.0-cp38-none-win_amd64.whl", hash = "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232"}, + {file = "rpds_py-0.20.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22"}, + {file = "rpds_py-0.20.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda"}, + {file = "rpds_py-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420"}, + {file = "rpds_py-0.20.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b"}, + {file = "rpds_py-0.20.0-cp39-none-win32.whl", hash = "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7"}, + {file = "rpds_py-0.20.0-cp39-none-win_amd64.whl", hash = "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344"}, + {file = "rpds_py-0.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec"}, + {file = "rpds_py-0.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8"}, + {file = "rpds_py-0.20.0.tar.gz", hash = "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121"}, +] + [[package]] name = "rsa" version = "4.9" @@ -3402,47 +3556,47 @@ files = [ [[package]] name = "tiktoken" -version = "0.6.0" +version = "0.7.0" description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" optional = true python-versions = ">=3.8" files = [ - {file = "tiktoken-0.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:277de84ccd8fa12730a6b4067456e5cf72fef6300bea61d506c09e45658d41ac"}, - {file = "tiktoken-0.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c44433f658064463650d61387623735641dcc4b6c999ca30bc0f8ba3fccaf5c"}, - {file = "tiktoken-0.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afb9a2a866ae6eef1995ab656744287a5ac95acc7e0491c33fad54d053288ad3"}, - {file = "tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c62c05b3109fefca26fedb2820452a050074ad8e5ad9803f4652977778177d9f"}, - {file = "tiktoken-0.6.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0ef917fad0bccda07bfbad835525bbed5f3ab97a8a3e66526e48cdc3e7beacf7"}, - {file = "tiktoken-0.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e095131ab6092d0769a2fda85aa260c7c383072daec599ba9d8b149d2a3f4d8b"}, - {file = "tiktoken-0.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:05b344c61779f815038292a19a0c6eb7098b63c8f865ff205abb9ea1b656030e"}, - {file = "tiktoken-0.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cefb9870fb55dca9e450e54dbf61f904aab9180ff6fe568b61f4db9564e78871"}, - {file = "tiktoken-0.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:702950d33d8cabc039845674107d2e6dcabbbb0990ef350f640661368df481bb"}, - {file = "tiktoken-0.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8d49d076058f23254f2aff9af603863c5c5f9ab095bc896bceed04f8f0b013a"}, - {file = "tiktoken-0.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:430bc4e650a2d23a789dc2cdca3b9e5e7eb3cd3935168d97d43518cbb1f9a911"}, - {file = "tiktoken-0.6.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:293cb8669757301a3019a12d6770bd55bec38a4d3ee9978ddbe599d68976aca7"}, - {file = "tiktoken-0.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7bd1a288b7903aadc054b0e16ea78e3171f70b670e7372432298c686ebf9dd47"}, - {file = "tiktoken-0.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac76e000183e3b749634968a45c7169b351e99936ef46f0d2353cd0d46c3118d"}, - {file = "tiktoken-0.6.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:17cc8a4a3245ab7d935c83a2db6bb71619099d7284b884f4b2aea4c74f2f83e3"}, - {file = "tiktoken-0.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:284aebcccffe1bba0d6571651317df6a5b376ff6cfed5aeb800c55df44c78177"}, - {file = "tiktoken-0.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c1a3a5d33846f8cd9dd3b7897c1d45722f48625a587f8e6f3d3e85080559be8"}, - {file = "tiktoken-0.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6318b2bb2337f38ee954fd5efa82632c6e5ced1d52a671370fa4b2eff1355e91"}, - {file = "tiktoken-0.6.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f5f0f2ed67ba16373f9a6013b68da298096b27cd4e1cf276d2d3868b5c7efd1"}, - {file = "tiktoken-0.6.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:75af4c0b16609c2ad02581f3cdcd1fb698c7565091370bf6c0cf8624ffaba6dc"}, - {file = "tiktoken-0.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:45577faf9a9d383b8fd683e313cf6df88b6076c034f0a16da243bb1c139340c3"}, - {file = "tiktoken-0.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7c1492ab90c21ca4d11cef3a236ee31a3e279bb21b3fc5b0e2210588c4209e68"}, - {file = "tiktoken-0.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e2b380c5b7751272015400b26144a2bab4066ebb8daae9c3cd2a92c3b508fe5a"}, - {file = "tiktoken-0.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f497598b9f58c99cbc0eb764b4a92272c14d5203fc713dd650b896a03a50ad"}, - {file = "tiktoken-0.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e65e8bd6f3f279d80f1e1fbd5f588f036b9a5fa27690b7f0cc07021f1dfa0839"}, - {file = "tiktoken-0.6.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5f1495450a54e564d236769d25bfefbf77727e232d7a8a378f97acddee08c1ae"}, - {file = "tiktoken-0.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6c4e4857d99f6fb4670e928250835b21b68c59250520a1941618b5b4194e20c3"}, - {file = "tiktoken-0.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:168d718f07a39b013032741867e789971346df8e89983fe3c0ef3fbd5a0b1cb9"}, - {file = "tiktoken-0.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:47fdcfe11bd55376785a6aea8ad1db967db7f66ea81aed5c43fad497521819a4"}, - {file = "tiktoken-0.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fb7d2ccbf1a7784810aff6b80b4012fb42c6fc37eaa68cb3b553801a5cc2d1fc"}, - {file = "tiktoken-0.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ccb7a111ee76af5d876a729a347f8747d5ad548e1487eeea90eaf58894b3138"}, - {file = "tiktoken-0.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2048e1086b48e3c8c6e2ceeac866561374cd57a84622fa49a6b245ffecb7744"}, - {file = "tiktoken-0.6.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07f229a5eb250b6403a61200199cecf0aac4aa23c3ecc1c11c1ca002cbb8f159"}, - {file = "tiktoken-0.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:432aa3be8436177b0db5a2b3e7cc28fd6c693f783b2f8722539ba16a867d0c6a"}, - {file = "tiktoken-0.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:8bfe8a19c8b5c40d121ee7938cd9c6a278e5b97dc035fd61714b4f0399d2f7a1"}, - {file = "tiktoken-0.6.0.tar.gz", hash = "sha256:ace62a4ede83c75b0374a2ddfa4b76903cf483e9cb06247f566be3bf14e6beed"}, + {file = "tiktoken-0.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485f3cc6aba7c6b6ce388ba634fbba656d9ee27f766216f45146beb4ac18b25f"}, + {file = "tiktoken-0.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e54be9a2cd2f6d6ffa3517b064983fb695c9a9d8aa7d574d1ef3c3f931a99225"}, + {file = "tiktoken-0.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79383a6e2c654c6040e5f8506f3750db9ddd71b550c724e673203b4f6b4b4590"}, + {file = "tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d4511c52caacf3c4981d1ae2df85908bd31853f33d30b345c8b6830763f769c"}, + {file = "tiktoken-0.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13c94efacdd3de9aff824a788353aa5749c0faee1fbe3816df365ea450b82311"}, + {file = "tiktoken-0.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8e58c7eb29d2ab35a7a8929cbeea60216a4ccdf42efa8974d8e176d50c9a3df5"}, + {file = "tiktoken-0.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:21a20c3bd1dd3e55b91c1331bf25f4af522c525e771691adbc9a69336fa7f702"}, + {file = "tiktoken-0.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:10c7674f81e6e350fcbed7c09a65bca9356eaab27fb2dac65a1e440f2bcfe30f"}, + {file = "tiktoken-0.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:084cec29713bc9d4189a937f8a35dbdfa785bd1235a34c1124fe2323821ee93f"}, + {file = "tiktoken-0.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:811229fde1652fedcca7c6dfe76724d0908775b353556d8a71ed74d866f73f7b"}, + {file = "tiktoken-0.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86b6e7dc2e7ad1b3757e8a24597415bafcfb454cebf9a33a01f2e6ba2e663992"}, + {file = "tiktoken-0.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1063c5748be36344c7e18c7913c53e2cca116764c2080177e57d62c7ad4576d1"}, + {file = "tiktoken-0.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:20295d21419bfcca092644f7e2f2138ff947a6eb8cfc732c09cc7d76988d4a89"}, + {file = "tiktoken-0.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:959d993749b083acc57a317cbc643fb85c014d055b2119b739487288f4e5d1cb"}, + {file = "tiktoken-0.7.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:71c55d066388c55a9c00f61d2c456a6086673ab7dec22dd739c23f77195b1908"}, + {file = "tiktoken-0.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:09ed925bccaa8043e34c519fbb2f99110bd07c6fd67714793c21ac298e449410"}, + {file = "tiktoken-0.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03c6c40ff1db0f48a7b4d2dafeae73a5607aacb472fa11f125e7baf9dce73704"}, + {file = "tiktoken-0.7.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d20b5c6af30e621b4aca094ee61777a44118f52d886dbe4f02b70dfe05c15350"}, + {file = "tiktoken-0.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d427614c3e074004efa2f2411e16c826f9df427d3c70a54725cae860f09e4bf4"}, + {file = "tiktoken-0.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8c46d7af7b8c6987fac9b9f61041b452afe92eb087d29c9ce54951280f899a97"}, + {file = "tiktoken-0.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:0bc603c30b9e371e7c4c7935aba02af5994a909fc3c0fe66e7004070858d3f8f"}, + {file = "tiktoken-0.7.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2398fecd38c921bcd68418675a6d155fad5f5e14c2e92fcf5fe566fa5485a858"}, + {file = "tiktoken-0.7.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8f5f6afb52fb8a7ea1c811e435e4188f2bef81b5e0f7a8635cc79b0eef0193d6"}, + {file = "tiktoken-0.7.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:861f9ee616766d736be4147abac500732b505bf7013cfaf019b85892637f235e"}, + {file = "tiktoken-0.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54031f95c6939f6b78122c0aa03a93273a96365103793a22e1793ee86da31685"}, + {file = "tiktoken-0.7.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:fffdcb319b614cf14f04d02a52e26b1d1ae14a570f90e9b55461a72672f7b13d"}, + {file = "tiktoken-0.7.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c72baaeaefa03ff9ba9688624143c858d1f6b755bb85d456d59e529e17234769"}, + {file = "tiktoken-0.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:131b8aeb043a8f112aad9f46011dced25d62629091e51d9dc1adbf4a1cc6aa98"}, + {file = "tiktoken-0.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cabc6dc77460df44ec5b879e68692c63551ae4fae7460dd4ff17181df75f1db7"}, + {file = "tiktoken-0.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8d57f29171255f74c0aeacd0651e29aa47dff6f070cb9f35ebc14c82278f3b25"}, + {file = "tiktoken-0.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ee92776fdbb3efa02a83f968c19d4997a55c8e9ce7be821ceee04a1d1ee149c"}, + {file = "tiktoken-0.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e215292e99cb41fbc96988ef62ea63bb0ce1e15f2c147a61acc319f8b4cbe5bf"}, + {file = "tiktoken-0.7.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8a81bac94769cab437dd3ab0b8a4bc4e0f9cf6835bcaa88de71f39af1791727a"}, + {file = "tiktoken-0.7.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d6d73ea93e91d5ca771256dfc9d1d29f5a554b83821a1dc0891987636e0ae226"}, + {file = "tiktoken-0.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:2bcb28ddf79ffa424f171dfeef9a4daff61a94c631ca6813f43967cb263b83b9"}, + {file = "tiktoken-0.7.0.tar.gz", hash = "sha256:1077266e949c24e0291f6c350433c6f0971365ece2b173a23bc3b9f9defef6b6"}, ] [package.dependencies] @@ -3951,4 +4105,4 @@ together = ["together"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "5dc9a83d0bf79d0fd7c4ee631af6381db64820612e76907ea635b9ee3b69d6ac" +content-hash = "bec2d7fbfb8a74921591b33c6128063d23166a8725e433e2e2991644781058c4" diff --git a/pyproject.toml b/pyproject.toml index 446e9da6..fe31f4bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ python-dotenv = "^1.0.0" backoff = "^2.2.1" langchain = {version = "<0.2.0", optional = true} magentic = {version = ">=0.17.0", optional = true, markers = "python_version >= '3.10'"} -litellm = {version = "^1.34.18", optional = true} +litellm = {version = "^1.41.12", optional = true} lamini = {version = "^2.1.8", optional = true} google-cloud-aiplatform = {version = ">=1.44.0", optional = true} mistralai = {version = "^0.1.5", optional = true} From 4c4f2b80b64d0dbbe02d33f1e10c21f5041149d8 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 22:25:10 -0400 Subject: [PATCH 09/13] Typo --- .github/workflows/test.yml | 4 ++-- ...bility.py => magentic_async_chat_perplexity.py} | 0 ...enai_async_logging_perplexity_compatibility.py} | 0 tests/conftest.py | 6 ++++-- tests/pytest.ini | 2 +- tests/test_openai_perplexity_compability.py | 14 +++++++------- 6 files changed, 14 insertions(+), 12 deletions(-) rename examples/logging/{magentic_async_chat_openai_perplexity_compability.py => magentic_async_chat_perplexity.py} (100%) rename examples/logging/{openai_async_logging_perplexity_compability.py => openai_async_logging_perplexity_compatibility.py} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4b96c0a..978ce338 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -119,7 +119,7 @@ jobs: echo "All variables are empty" poetry run pytest -vv tests/ --ignore=tests/test_cli.py poetry run pytest --llm_provider=anthropic -vv tests/test_magentic.py - poetry run pytest --llm_provider=litellm --openai_compability_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat + poetry run pytest --llm_provider=litellm --openai_compatibility_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat fi - name: Run scheduled llm tests @@ -128,4 +128,4 @@ jobs: echo "This is a schedule event" poetry run pytest -vv tests/ --ignore=tests/test_cli.py poetry run pytest --openai_model=gpt-4o -m chat -vv tests/test_openai.py - poetry run pytest --llm_provider=litellm --openai_compability_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat + poetry run pytest --llm_provider=litellm --openai_compatibility_model=perplexity/llama-3.1-sonar-small-128k-chat -vv tests/test_magentic.py -m chat diff --git a/examples/logging/magentic_async_chat_openai_perplexity_compability.py b/examples/logging/magentic_async_chat_perplexity.py similarity index 100% rename from examples/logging/magentic_async_chat_openai_perplexity_compability.py rename to examples/logging/magentic_async_chat_perplexity.py diff --git a/examples/logging/openai_async_logging_perplexity_compability.py b/examples/logging/openai_async_logging_perplexity_compatibility.py similarity index 100% rename from examples/logging/openai_async_logging_perplexity_compability.py rename to examples/logging/openai_async_logging_perplexity_compatibility.py diff --git a/tests/conftest.py b/tests/conftest.py index 809a9b4b..d4750838 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,9 @@ def pytest_addoption(parser): parser.addoption("--llm_provider", action="store", help="Model provider name for Magentic tests") parser.addoption( - "--openai_compability_model", action="store", help="Model name for client compability model in Magentic tests" + "--openai_compatibility_model", + action="store", + help="Model name for client compatibility model in Magentic tests", ) @@ -73,7 +75,7 @@ def magentic_models(request): model_configs_to_providers = { "openai": ["openai_model", "openai_vision_model"], "anthropic": ["anthropic_model", "anthropic_model"], - "litellm": ["openai_compability_model", "openai_compability_model"], + "litellm": ["openai_compatibility_model", "openai_compatibility_model"], } model_configs = model_configs_to_providers[llm_provider] diff --git a/tests/pytest.ini b/tests/pytest.ini index f655cc15..d614c2de 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -7,7 +7,7 @@ addopts = --google_model=gemini-1.5-pro-latest --mistralai_model=mistral-tiny --lamini_model=meta-llama/Llama-2-7b-chat-hf - --openai_compability_model=perplexity/llama-3.1-sonar-small-128k-chat + --openai_compatibility_model=perplexity/llama-3.1-sonar-small-128k-chat --llm_provider=openai markers = diff --git a/tests/test_openai_perplexity_compability.py b/tests/test_openai_perplexity_compability.py index c78c5444..447db81d 100644 --- a/tests/test_openai_perplexity_compability.py +++ b/tests/test_openai_perplexity_compability.py @@ -16,7 +16,7 @@ if "PERPLEXITYAI_API_KEY" not in os.environ: raise ValueError("Please set the PERPLEXITYAI_API_KEY environment variable.") -compability_config = { +compatibility_config = { "base_url": "https://api.perplexity.ai", "api_key": os.environ.get("PERPLEXITYAI_API_KEY"), } @@ -24,7 +24,7 @@ @pytest.mark.chat def test_chat(session): - client = openai.OpenAI(**compability_config) + client = openai.OpenAI(**compatibility_config) completion = client.chat.completions.create( model=model_name, messages=[ @@ -47,7 +47,7 @@ def test_chat(session): @pytest.mark.chat def test_chat_not_given(session): - client = openai.OpenAI(**compability_config) + client = openai.OpenAI(**compatibility_config) completion = client.chat.completions.create( model=model_name, messages=[ @@ -70,7 +70,7 @@ def test_chat_not_given(session): @pytest.mark.async_client @pytest.mark.asyncio(scope="module") async def test_chat_async(session): - client = AsyncOpenAI(**compability_config) + client = AsyncOpenAI(**compatibility_config) completion = await client.chat.completions.create( model=model_name, messages=[{"role": "user", "content": "Say this is a test"}], @@ -86,7 +86,7 @@ async def test_chat_async(session): @pytest.mark.async_client @pytest.mark.asyncio(scope="module") async def test_perplexity_chat_async(session): - client = AsyncOpenAI(**compability_config) + client = AsyncOpenAI(**compatibility_config) completion = await client.chat.completions.create( model=model_name, messages=[{"role": "user", "content": "Say this is a test"}], @@ -101,7 +101,7 @@ async def test_perplexity_chat_async(session): @pytest.mark.chat @pytest.mark.stream def test_chat_stream(session): - client = openai.OpenAI(**compability_config) + client = openai.OpenAI(**compatibility_config) response = client.chat.completions.create( model=model_name, messages=[{"role": "user", "content": "Count to 5"}], @@ -120,7 +120,7 @@ def test_chat_stream(session): @pytest.mark.stream @pytest.mark.asyncio(scope="module") async def test_chat_async_stream(session): - client = AsyncOpenAI(**compability_config) + client = AsyncOpenAI(**compatibility_config) output = "" stream = await client.chat.completions.create( From 6bc485d83145ed2e6c3c95250993c8800c314fbd Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 22:26:14 -0400 Subject: [PATCH 10/13] Rename example logging file --- ...atibility.py => perplexity_async_chat_openai_compatibility.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/logging/{openai_async_logging_perplexity_compatibility.py => perplexity_async_chat_openai_compatibility.py} (100%) diff --git a/examples/logging/openai_async_logging_perplexity_compatibility.py b/examples/logging/perplexity_async_chat_openai_compatibility.py similarity index 100% rename from examples/logging/openai_async_logging_perplexity_compatibility.py rename to examples/logging/perplexity_async_chat_openai_compatibility.py From 37e6d89f26b26d443b926ec279ca5ae16b247627 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 22:35:40 -0400 Subject: [PATCH 11/13] Add comment on openai path list --- log10/_httpx_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index b780f5a5..ab7af4dd 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -38,6 +38,8 @@ class LLM_CLIENTS(Enum): PROVIDER_PATHS = { LLM_CLIENTS.ANTHROPIC: ["/v1/messages", "/v1/complete"], + # OpenAI and Mistral use the path "v1/chat/completions" + # Perplexity uses the path "chat/completions". Documentation: https://docs.perplexity.ai/reference/post_chat_completions LLM_CLIENTS.OPENAI: ["v1/chat/completions", "chat/completions"], } From 2b5bbac569cc6bc139e3f6c0e7aa5bd4ec82b2fc Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 22:44:21 -0400 Subject: [PATCH 12/13] Update llm_provider -> llm_client for consistency --- log10/_httpx_utils.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index ab7af4dd..490b4227 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -36,7 +36,7 @@ class LLM_CLIENTS(Enum): UNKNOWN = "Unknown" -PROVIDER_PATHS = { +CLIENT_PATHS = { LLM_CLIENTS.ANTHROPIC: ["/v1/messages", "/v1/complete"], # OpenAI and Mistral use the path "v1/chat/completions" # Perplexity uses the path "chat/completions". Documentation: https://docs.perplexity.ai/reference/post_chat_completions @@ -247,13 +247,19 @@ def format_anthropic_request(request_content) -> str: return json.dumps(request_content) -def _get_llm_provider(request: Request) -> LLM_CLIENTS: +def _get_llm_client(request: Request) -> LLM_CLIENTS: + """ + The request object includes the user-agent header, which is used to identify the LLM client. + For example: + - headers({'user-agent': 'AsyncOpenAI/Python 1.40.6'}) + - headers({'user-agent': 'Anthropic/Python 0.34.0'}) + """ user_agent = request.headers.get("user-agent", "") class_name = user_agent.split("/")[0] if class_name in ["AsyncAnthropic", "Anthropic"]: return LLM_CLIENTS.ANTHROPIC - elif class_name in ["AsyncOpenAI"]: + elif class_name in ["AsyncOpenAI", "OpenAI"]: return LLM_CLIENTS.OPENAI else: return LLM_CLIENTS.UNKNOWN @@ -265,9 +271,9 @@ def _init_log_row(request: Request): orig_module = "" orig_qualname = "" request_content_decode = request.content.decode("utf-8") - llm_provider = _get_llm_provider(request) + llm_client = _get_llm_client(request) - if llm_provider == LLM_CLIENTS.OPENAI: + if llm_client == LLM_CLIENTS.OPENAI: if "chat" in str(request.url): kind = "chat" orig_module = "openai.api_resources.chat_completion" @@ -276,7 +282,7 @@ def _init_log_row(request: Request): kind = "completion" orig_module = "openai.api_resources.completion" orig_qualname = "Completion.create" - elif llm_provider == LLM_CLIENTS.ANTHROPIC: + elif llm_client == LLM_CLIENTS.ANTHROPIC: kind = "chat" url_path = request.url content_type = request.headers.get("content-type") @@ -313,16 +319,14 @@ def _init_log_row(request: Request): def get_completion_id(request: Request): - llm_provider = _get_llm_provider(request) - if llm_provider is LLM_CLIENTS.UNKNOWN: + llm_client = _get_llm_client(request) + if llm_client is LLM_CLIENTS.UNKNOWN: logger.debug("Currently logging is only available for async openai and anthropic.") return # Check if the request URL matches any of the allowed paths for the class name - if not any(path in str(request.url) for path in PROVIDER_PATHS.get(llm_provider, [])): - logger.debug( - f"Currently logging is only available for {llm_provider} {', '.join(PROVIDER_PATHS[llm_provider])}." - ) + if not any(path in str(request.url) for path in CLIENT_PATHS.get(llm_client, [])): + logger.debug(f"Currently logging is only available for {llm_client} {', '.join(CLIENT_PATHS[llm_client])}.") return completion_id = str(uuid.uuid4()) @@ -440,7 +444,7 @@ async def log_request(self, request: httpx.Request): class _LogResponse(Response): def __init__(self, *args, **kwargs): self.log_row = kwargs.pop("log_row") - self.llm_provider = _get_llm_provider(kwargs.get("request")) + self.llm_client = _get_llm_client(kwargs.get("request")) self.host_header = kwargs.get("request").headers.get("host") super().__init__(*args, **kwargs) @@ -457,7 +461,7 @@ def patch_streaming_log(self, duration: int, full_response: str): ] separator = ( - "\r\n\r\n" if self.llm_provider == LLM_CLIENTS.OPENAI and "perplexity" in self.host_header else "\n\n" + "\r\n\r\n" if self.llm_client == LLM_CLIENTS.OPENAI and "perplexity" in self.host_header else "\n\n" ) responses = full_response.split(separator) response_json = self.parse_response_data(responses) @@ -502,9 +506,9 @@ async def aiter_bytes(self, *args, **kwargs): yield chunk def is_response_end_reached(self, text: str) -> bool: - if self.llm_provider == LLM_CLIENTS.ANTHROPIC: + if self.llm_client == LLM_CLIENTS.ANTHROPIC: return self.is_anthropic_response_end_reached(text) - elif self.llm_provider == LLM_CLIENTS.OPENAI: + elif self.llm_client == LLM_CLIENTS.OPENAI: if "perplexity" in self.host_header: return self.is_perplexity_response_end_reached(text) else: @@ -690,9 +694,9 @@ def parse_openai_responses(self, responses: list[str]): return response_json def parse_response_data(self, responses: list[str]): - if self.llm_provider == LLM_CLIENTS.ANTHROPIC: + if self.llm_client == LLM_CLIENTS.ANTHROPIC: return self.parse_anthropic_responses(responses) - elif self.llm_provider == LLM_CLIENTS.OPENAI: + elif self.llm_client == LLM_CLIENTS.OPENAI: return self.parse_openai_responses(responses) else: logger.debug("Currently logging is only available for async openai and anthropic.") From 676dc31b1b40072c8209dd4ed949b705f7734804 Mon Sep 17 00:00:00 2001 From: Kim Tran Date: Thu, 22 Aug 2024 22:55:35 -0400 Subject: [PATCH 13/13] Feedback --- log10/_httpx_utils.py | 13 +++++++------ ..._compability.py => test_openai_compatibility.py} | 0 2 files changed, 7 insertions(+), 6 deletions(-) rename tests/{test_openai_perplexity_compability.py => test_openai_compatibility.py} (100%) diff --git a/log10/_httpx_utils.py b/log10/_httpx_utils.py index 490b4227..5fa05620 100644 --- a/log10/_httpx_utils.py +++ b/log10/_httpx_utils.py @@ -257,10 +257,8 @@ def _get_llm_client(request: Request) -> LLM_CLIENTS: user_agent = request.headers.get("user-agent", "") class_name = user_agent.split("/")[0] - if class_name in ["AsyncAnthropic", "Anthropic"]: - return LLM_CLIENTS.ANTHROPIC - elif class_name in ["AsyncOpenAI", "OpenAI"]: - return LLM_CLIENTS.OPENAI + if class_name in USER_AGENT_NAME_TO_PROVIDER.keys(): + return USER_AGENT_NAME_TO_PROVIDER[class_name] else: return LLM_CLIENTS.UNKNOWN @@ -528,8 +526,11 @@ def is_perplexity_response_end_reached(self, text: str): return last_object.get("choices", [{}])[0].get("finish_reason", "") == "stop" def is_openai_response_end_reached(self, text: str): - # For perplexity, the last item in the responses is empty - return "data: [DONE]" in text or not text + """ + In Perplexity, the last item in the responses is empty. + In OpenAI and Mistral, the last item in the responses is "data: [DONE]". + """ + return not text or "data: [DONE]" in text def parse_anthropic_responses(self, responses: list[str]): message_id = "" diff --git a/tests/test_openai_perplexity_compability.py b/tests/test_openai_compatibility.py similarity index 100% rename from tests/test_openai_perplexity_compability.py rename to tests/test_openai_compatibility.py