Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add vlm document #1866

Merged
merged 22 commits into from
Nov 1, 2024
Merged

Add vlm document #1866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
a8f4611
change file tree
Oct 31, 2024
99300ec
add all files
Oct 31, 2024
08a801d
fix empty ipynb
Oct 31, 2024
d8a8e32
Merge branch 'main' into change_file_tree
zhaochenyang20 Oct 31, 2024
1fde98b
Merge branch 'change_file_tree' of github.com:sgl-project/sglang into…
Oct 31, 2024
e696a56
improve CI and remove empty files
Oct 31, 2024
21cc435
improve CI and remove empty files
Oct 31, 2024
3bf7ece
fix make compile
Nov 1, 2024
be14585
Merge branch 'main' into change_file_tree
zhaochenyang20 Nov 1, 2024
4f87ebc
Merge branch 'change_file_tree' of github.com:sgl-project/sglang into…
Nov 1, 2024
56c958f
add make compile
Nov 1, 2024
a24968b
use make clean'
Nov 1, 2024
7ac5649
keep manual deploy
Nov 1, 2024
5f76154
update readme
Nov 1, 2024
181f336
fix lint
Nov 1, 2024
5db9438
add vlm doc
Nov 1, 2024
879a6e6
add vlm
Nov 1, 2024
e2cf27d
Merge branch 'main' of github.com:sgl-project/sglang
Nov 1, 2024
b4884d9
fetch main
Nov 1, 2024
88952ca
fix conflicts
Nov 1, 2024
525925b
Merge branch 'main' into add_vlm
zhaochenyang20 Nov 1, 2024
f572e1d
Merge branch 'add_vlm' of github.com:sgl-project/sglang into add_vlm
Nov 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 35 additions & 157 deletions docs/backend/embedding_model.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"## Launch A Server\n",
"\n",
"The following code is equivalent to running this in the shell:\n",
"\n",
"```bash\n",
"python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-7B-instruct \\\n",
" --port 30010 --host 0.0.0.0 --is-embedding\n",
Expand All @@ -44,161 +45,38 @@
"output_type": "stream",
"text": [
"/home/chenyang/miniconda3/envs/AlphaMeemory/lib/python3.11/site-packages/transformers/utils/hub.py:128: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n",
" warnings.warn(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:37] server_args=ServerArgs(model_path='Alibaba-NLP/gte-Qwen2-7B-instruct', tokenizer_path='Alibaba-NLP/gte-Qwen2-7B-instruct', tokenizer_mode='auto', skip_tokenizer_init=False, load_format='auto', trust_remote_code=False, dtype='auto', kv_cache_dtype='auto', quantization=None, context_length=None, device='cuda', served_model_name='Alibaba-NLP/gte-Qwen2-7B-instruct', chat_template=None, is_embedding=True, host='0.0.0.0', port=30010, mem_fraction_static=0.88, max_running_requests=None, max_total_tokens=None, chunked_prefill_size=8192, max_prefill_tokens=16384, schedule_policy='lpm', schedule_conservativeness=1.0, tp_size=1, stream_interval=1, random_seed=314021918, constrained_json_whitespace_pattern=None, decode_log_interval=40, log_level='info', log_level_http=None, log_requests=False, show_time_cost=False, api_key=None, file_storage_pth='SGLang_storage', enable_cache_report=False, watchdog_timeout=600, dp_size=1, load_balance_method='round_robin', dist_init_addr=None, nnodes=1, node_rank=0, json_model_override_args='{}', enable_double_sparsity=False, ds_channel_config_path=None, ds_heavy_channel_num=32, ds_heavy_token_num=256, ds_heavy_channel_type='qk', ds_sparse_decode_threshold=4096, lora_paths=None, max_loras_per_batch=8, attention_backend='flashinfer', sampling_backend='flashinfer', grammar_backend='outlines', disable_flashinfer=False, disable_flashinfer_sampling=False, disable_radix_cache=False, disable_regex_jump_forward=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, disable_disk_cache=False, disable_custom_all_reduce=False, disable_mla=False, disable_penalizer=False, disable_nan_detection=False, enable_overlap_schedule=False, enable_mixed_chunk=False, enable_torch_compile=False, torch_compile_max_bs=32, cuda_graph_max_bs=160, torchao_config='', enable_p2p_check=False, triton_attention_reduce_in_fp32=False, num_continuous_decode_steps=1)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" warnings.warn(\n",
"[2024-10-31 22:40:37] server_args=ServerArgs(model_path='Alibaba-NLP/gte-Qwen2-7B-instruct', tokenizer_path='Alibaba-NLP/gte-Qwen2-7B-instruct', tokenizer_mode='auto', skip_tokenizer_init=False, load_format='auto', trust_remote_code=False, dtype='auto', kv_cache_dtype='auto', quantization=None, context_length=None, device='cuda', served_model_name='Alibaba-NLP/gte-Qwen2-7B-instruct', chat_template=None, is_embedding=True, host='0.0.0.0', port=30010, mem_fraction_static=0.88, max_running_requests=None, max_total_tokens=None, chunked_prefill_size=8192, max_prefill_tokens=16384, schedule_policy='lpm', schedule_conservativeness=1.0, tp_size=1, stream_interval=1, random_seed=309155486, constrained_json_whitespace_pattern=None, decode_log_interval=40, log_level='info', log_level_http=None, log_requests=False, show_time_cost=False, api_key=None, file_storage_pth='SGLang_storage', enable_cache_report=False, watchdog_timeout=600, dp_size=1, load_balance_method='round_robin', dist_init_addr=None, nnodes=1, node_rank=0, json_model_override_args='{}', enable_double_sparsity=False, ds_channel_config_path=None, ds_heavy_channel_num=32, ds_heavy_token_num=256, ds_heavy_channel_type='qk', ds_sparse_decode_threshold=4096, lora_paths=None, max_loras_per_batch=8, attention_backend='flashinfer', sampling_backend='flashinfer', grammar_backend='outlines', disable_flashinfer=False, disable_flashinfer_sampling=False, disable_radix_cache=False, disable_regex_jump_forward=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, disable_disk_cache=False, disable_custom_all_reduce=False, disable_mla=False, disable_penalizer=False, disable_nan_detection=False, enable_overlap_schedule=False, enable_mixed_chunk=False, enable_torch_compile=False, torch_compile_max_bs=32, cuda_graph_max_bs=160, torchao_config='', enable_p2p_check=False, triton_attention_reduce_in_fp32=False, num_continuous_decode_steps=1)\n",
"/home/chenyang/miniconda3/envs/AlphaMeemory/lib/python3.11/site-packages/transformers/utils/hub.py:128: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n",
" warnings.warn(\n",
"/home/chenyang/miniconda3/envs/AlphaMeemory/lib/python3.11/site-packages/transformers/utils/hub.py:128: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.\n",
" warnings.warn(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:43 TP0] Init torch distributed begin.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:44 TP0] Load weight begin. avail mem=47.27 GB\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:44 TP0] lm_eval is not installed, GPTQ may not be usable\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"INFO 10-31 19:47:45 weight_utils.py:243] Using model weights format ['*.safetensors']\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 0% Completed | 0/7 [00:00<?, ?it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 14% Completed | 1/7 [00:00<00:03, 1.96it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 29% Completed | 2/7 [00:01<00:03, 1.39it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 43% Completed | 3/7 [00:02<00:03, 1.13it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 57% Completed | 4/7 [00:03<00:02, 1.00it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 71% Completed | 5/7 [00:04<00:02, 1.05s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 86% Completed | 6/7 [00:05<00:01, 1.09s/it]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r",
"Loading safetensors checkpoint shards: 100% Completed | 7/7 [00:07<00:00, 1.11s/it]\n",
"\r",
"Loading safetensors checkpoint shards: 100% Completed | 7/7 [00:07<00:00, 1.01s/it]\n",
" warnings.warn(\n",
"[2024-10-31 22:40:42 TP0] Init torch distributed begin.\n",
"[2024-10-31 22:40:43 TP0] Load weight begin. avail mem=47.27 GB\n",
"[2024-10-31 22:40:43 TP0] lm_eval is not installed, GPTQ may not be usable\n",
"INFO 10-31 22:40:44 weight_utils.py:243] Using model weights format ['*.safetensors']\n",
"Loading safetensors checkpoint shards: 0% Completed | 0/7 [00:00<?, ?it/s]\n",
"Loading safetensors checkpoint shards: 14% Completed | 1/7 [00:00<00:03, 1.97it/s]\n",
"Loading safetensors checkpoint shards: 29% Completed | 2/7 [00:01<00:03, 1.40it/s]\n",
"Loading safetensors checkpoint shards: 43% Completed | 3/7 [00:02<00:03, 1.11it/s]\n",
"Loading safetensors checkpoint shards: 57% Completed | 4/7 [00:03<00:03, 1.00s/it]\n",
"Loading safetensors checkpoint shards: 71% Completed | 5/7 [00:04<00:02, 1.07s/it]\n",
"Loading safetensors checkpoint shards: 86% Completed | 6/7 [00:05<00:01, 1.10s/it]\n",
"Loading safetensors checkpoint shards: 100% Completed | 7/7 [00:07<00:00, 1.12s/it]\n",
"Loading safetensors checkpoint shards: 100% Completed | 7/7 [00:07<00:00, 1.02s/it]\n",
"\n",
"[2024-10-31 19:47:53 TP0] Load weight end. type=Qwen2ForCausalLM, dtype=torch.float16, avail mem=32.91 GB\n",
"[2024-10-31 19:47:53 TP0] Memory pool end. avail mem=4.56 GB\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:53 TP0] max_total_num_tokens=509971, max_prefill_tokens=16384, max_running_requests=2049, context_len=131072\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:54] INFO: Started server process [1552642]\n",
"[2024-10-31 19:47:54] INFO: Waiting for application startup.\n",
"[2024-10-31 19:47:54] INFO: Application startup complete.\n",
"[2024-10-31 19:47:54] INFO: Uvicorn running on http://0.0.0.0:30010 (Press CTRL+C to quit)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:54] INFO: 127.0.0.1:47776 - \"GET /v1/models HTTP/1.1\" 200 OK\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:55] INFO: 127.0.0.1:50344 - \"GET /get_model_info HTTP/1.1\" 200 OK\n",
"[2024-10-31 19:47:55 TP0] Prefill batch. #new-seq: 1, #new-token: 6, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:55] INFO: 127.0.0.1:50352 - \"POST /encode HTTP/1.1\" 200 OK\n",
"[2024-10-31 19:47:55] The server is fired up and ready to roll!\n"
"[2024-10-31 22:40:51 TP0] Load weight end. type=Qwen2ForCausalLM, dtype=torch.float16, avail mem=32.91 GB\n",
"[2024-10-31 22:40:51 TP0] Memory pool end. avail mem=4.56 GB\n",
"[2024-10-31 22:40:52 TP0] max_total_num_tokens=509971, max_prefill_tokens=16384, max_running_requests=2049, context_len=131072\n",
"[2024-10-31 22:40:52] INFO: Started server process [1752367]\n",
"[2024-10-31 22:40:52] INFO: Waiting for application startup.\n",
"[2024-10-31 22:40:52] INFO: Application startup complete.\n",
"[2024-10-31 22:40:52] INFO: Uvicorn running on http://0.0.0.0:30010 (Press CTRL+C to quit)\n",
"[2024-10-31 22:40:52] INFO: 127.0.0.1:41676 - \"GET /v1/models HTTP/1.1\" 200 OK\n",
"[2024-10-31 22:40:53] INFO: 127.0.0.1:41678 - \"GET /get_model_info HTTP/1.1\" 200 OK\n",
"[2024-10-31 22:40:53 TP0] Prefill batch. #new-seq: 1, #new-token: 6, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0\n",
"[2024-10-31 22:40:54] INFO: 127.0.0.1:41684 - \"POST /encode HTTP/1.1\" 200 OK\n",
"[2024-10-31 22:40:54] The server is fired up and ready to roll!\n"
]
},
{
Expand Down Expand Up @@ -255,8 +133,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:59 TP0] Prefill batch. #new-seq: 1, #new-token: 4, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0\n",
"[2024-10-31 19:47:59] INFO: 127.0.0.1:50358 - \"POST /v1/embeddings HTTP/1.1\" 200 OK\n"
"[2024-10-31 22:40:57 TP0] Prefill batch. #new-seq: 1, #new-token: 4, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0\n",
"[2024-10-31 22:40:57] INFO: 127.0.0.1:51746 - \"POST /v1/embeddings HTTP/1.1\" 200 OK\n"
]
},
{
Expand Down Expand Up @@ -312,8 +190,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:47:59 TP0] Prefill batch. #new-seq: 1, #new-token: 1, #cached-token: 3, cache hit rate: 21.43%, token usage: 0.00, #running-req: 0, #queue-req: 0\n",
"[2024-10-31 19:47:59] INFO: 127.0.0.1:50362 - \"POST /v1/embeddings HTTP/1.1\" 200 OK\n"
"[2024-10-31 22:40:58 TP0] Prefill batch. #new-seq: 1, #new-token: 1, #cached-token: 3, cache hit rate: 21.43%, token usage: 0.00, #running-req: 0, #queue-req: 0\n",
"[2024-10-31 22:40:58] INFO: 127.0.0.1:51750 - \"POST /v1/embeddings HTTP/1.1\" 200 OK\n"
]
},
{
Expand Down Expand Up @@ -377,8 +255,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"[2024-10-31 19:48:01 TP0] Prefill batch. #new-seq: 1, #new-token: 1, #cached-token: 3, cache hit rate: 33.33%, token usage: 0.00, #running-req: 0, #queue-req: 0\n",
"[2024-10-31 19:48:01] INFO: 127.0.0.1:50366 - \"POST /v1/embeddings HTTP/1.1\" 200 OK\n"
"[2024-10-31 22:41:00 TP0] Prefill batch. #new-seq: 1, #new-token: 1, #cached-token: 3, cache hit rate: 33.33%, token usage: 0.00, #running-req: 0, #queue-req: 0\n",
"[2024-10-31 22:41:00] INFO: 127.0.0.1:51762 - \"POST /v1/embeddings HTTP/1.1\" 200 OK\n"
]
},
{
Expand Down
Loading
Loading