From 8700ba9f042872885b989ef138b44befad6ea4e0 Mon Sep 17 00:00:00 2001
From: Michael Goin <michael@neuralmagic.com>
Date: Thu, 15 Aug 2024 21:26:19 -0400
Subject: [PATCH] [Bugfix][Harmless] Fix float16 dtype for model_is_embedding
 (#7566)

---
 vllm/entrypoints/openai/api_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py
index d89b87534320b..ad5ba4a93ba63 100644
--- a/vllm/entrypoints/openai/api_server.py
+++ b/vllm/entrypoints/openai/api_server.py
@@ -66,7 +66,7 @@ def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool:
                        tokenizer_mode="auto",
                        trust_remote_code=trust_remote_code,
                        seed=0,
-                       dtype="float16").embedding_mode
+                       dtype="auto").embedding_mode
 
 
 @asynccontextmanager