Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add image_token in conversation.py #1632

Merged
merged 3 commits into from
Oct 11, 2024
Merged

Add image_token in conversation.py #1632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c4ccd03
add image_token in conversation.py
merrymercy Oct 11, 2024
d6bee31
Update conversation.py
merrymercy Oct 11, 2024
f4f20dc
update
merrymercy Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions python/sglang/srt/conversation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ class Conversation:
sep2: str = None
# Stop criteria (the default one is EOS token)
stop_str: Union[str, List[str]] = None
# The string that represents an image token in the prompt
image_token: str = "<image>"

image_data: Optional[List[str]] = None
modalities: Optional[List[str]] = None

Expand Down Expand Up @@ -334,6 +337,7 @@ def copy(self):
sep=self.sep,
sep2=self.sep2,
stop_str=self.stop_str,
image_token=self.image_token,
)

def dict(self):
Expand Down Expand Up @@ -381,6 +385,7 @@ def generate_chat_conv(
stop_str=conv.stop_str,
image_data=[],
modalities=[],
image_token=conv.image_token,
)

if isinstance(request.messages, str):
Expand Down Expand Up @@ -412,9 +417,13 @@ def generate_chat_conv(
num_image_url += 1
conv.modalities.append(content.modalities)
if num_image_url > 1:
image_token = "<image>"
image_token = conv.image_token
else:
image_token = "<image>\n"
image_token = (
conv.image_token + "\n"
if conv.name != "qwen2-vl"
else conv.image_token
)
for content in message.content:
if content.type == "text":
if num_image_url > 16:
Expand Down
4 changes: 3 additions & 1 deletion python/sglang/srt/openai_api/adapter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ def create_streaming_error_response(
def load_chat_template_for_openai_api(tokenizer_manager, chat_template_arg):
global chat_template_name

logger.info(f"Use chat template: {chat_template_arg}")
logger.info(
f"Use chat template for the OpenAI-compatible API server: {chat_template_arg}"
)
if not chat_template_exists(chat_template_arg):
if not os.path.exists(chat_template_arg):
raise RuntimeError(
Expand Down
Loading