Skip to content

Commit

Permalink
[Misc] Update Pixtral example (vllm-project#8431)
Browse files Browse the repository at this point in the history
  • Loading branch information
ywang96 authored and Jeffwan committed Sep 19, 2024
1 parent 978da5e commit a17d96f
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions examples/offline_inference_pixtral.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# - Server:
#
# ```bash
# vllm serve mistralai/Pixtral-12B-2409 --tokenizer_mode mistral --limit_mm_per_prompt 'image=4' --max_num_batched_tokens 16384
# vllm serve mistralai/Pixtral-12B-2409 --tokenizer-mode mistral --limit-mm-per-prompt 'image=4' --max-model-len 16384
# ```
#
# - Client:
Expand Down Expand Up @@ -45,6 +45,7 @@ def run_simple_demo():
model_name = "mistralai/Pixtral-12B-2409"
sampling_params = SamplingParams(max_tokens=8192)

# Lower max_num_seqs or max_model_len on low-VRAM GPUs.
llm = LLM(model=model_name, tokenizer_mode="mistral")

prompt = "Describe this image in one sentence."
Expand Down Expand Up @@ -83,7 +84,7 @@ def run_advanced_demo():
model=model_name,
tokenizer_mode="mistral",
limit_mm_per_prompt={"image": max_img_per_msg},
max_num_batched_tokens=max_img_per_msg * max_tokens_per_img,
max_model_len=max_img_per_msg * max_tokens_per_img,
)

prompt = "Describe the following image."
Expand Down

0 comments on commit a17d96f

Please sign in to comment.