From 9bad6e1a6e1f90f28ad0ac8305a58223da6e7acf Mon Sep 17 00:00:00 2001 From: Bagatur Date: Thu, 19 Dec 2024 13:20:10 -0800 Subject: [PATCH] xfail openai image token count test --- .../tests/integration_tests/chat_models/test_base.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py index f2490f89501f8..427a030779e84 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py @@ -706,6 +706,12 @@ async def test_openai_response_headers_async() -> None: assert "content-type" in headers +@pytest.mark.xfail( + reason=( + "As of 12.19.24 OpenAI API returns 1151 instead of 1118. Not clear yet if " + "this is an undocumented API change or a bug on their end." + ) +) def test_image_token_counting_jpeg() -> None: model = ChatOpenAI(model="gpt-4o", temperature=0) image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" @@ -738,6 +744,12 @@ def test_image_token_counting_jpeg() -> None: assert expected == actual +@pytest.mark.xfail( + reason=( + "As of 12.19.24 OpenAI API returns 871 instead of 779. Not clear yet if " + "this is an undocumented API change or a bug on their end." + ) +) def test_image_token_counting_png() -> None: model = ChatOpenAI(model="gpt-4o", temperature=0) image_url = "https://upload.wikimedia.org/wikipedia/commons/4/47/PNG_transparency_demonstration_1.png"