Merge branch 'sgl-project:main' into main

haichuan1221 · Jul 30, 2024 · 7081dc9 · 7081dc9
2 parents 2964c8e + 1edd4e0
commit 7081dc9
Show file tree

Hide file tree

Showing 11 changed files with 67 additions and 27 deletions.
diff --git a/.github/workflows/pr-e2e-test.yml b/.github/workflows/pr-e2e-test.yml
@@ -3,8 +3,12 @@ name: PR E2E Test
 on:
   push:
     branches: [ main ]
+    paths:
+      - "python/sglang/*"
   pull_request:
     branches: [ main ]
+    paths:
+      - "python/sglang/*"
   workflow_dispatch:
 
 jobs:
@@ -26,22 +30,22 @@ jobs:
 
     - name: Launch server and run benchmark
       run: |
-        python3 -m sglang.launch_server --model /home/lmzheng/zhyncs/Meta-Llama-3.1-8B-Instruct --port 8413 &
+        python3 -m sglang.launch_server --model /home/lmzheng/zhyncs/Meta-Llama-3.1-8B-Instruct --port 8413 --disable-radix-cache &
 
         echo "Waiting for server to start..."
-        for i in {1..60}; do
+        for i in {1..120}; do
           if curl -s http://127.0.0.1:8413/health; then
             echo "Server is up!"
             break
           fi
-          if [ $i -eq 60 ]; then
-            echo "Server failed to start within 60 seconds"
+          if [ $i -eq 120 ]; then
+            echo "Server failed to start within 120 seconds"
             exit 1
           fi
           sleep 1
         done
 
-        python3 -m sglang.bench_serving --backend sglang --port 8413
+        cd /home/lmzheng/zhyncs && python3 -m sglang.bench_serving --backend sglang --port 8413 --dataset-name random --num-prompts 3000 --random-input 256 --random-output 512
 
         echo "Stopping server..."
         kill -9 $(ps aux | grep sglang | grep Meta-Llama-3.1-8B-Instruct | grep -v grep | awk '{print $2}')
diff --git a/README.md b/README.md
@@ -54,8 +54,8 @@ pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/
 
 ### Method 2: From source
 ```
-# Use the stable rel branch
-git clone -b rel https://github.com/sgl-project/sglang.git
+# Use the stable release branch
+git clone -b release https://github.com/sgl-project/sglang.git
 cd sglang
 
 pip install --upgrade pip

diff --git a/benchmark/blog_v0_2/README.md b/benchmark/blog_v0_2/README.md
@@ -7,7 +7,7 @@
 ```bash
 git clone https://github.com/sgl-project/sglang.git
 cd sglang
-git checkout v0.2.5
+git checkout v0.2.7
 
 pip install --upgrade pip
 pip install -e "python[all]"

diff --git a/docs/en/setup_runner.md b/docs/en/setup_runner.md
@@ -0,0 +1,29 @@
+# Set up self hosted runner for GitHub Action
+
+## Config Runner
+
+```bash
+# https://github.com/sgl-project/sglang/settings/actions/runners/new?arch=x64&os=linux
+# Involves some TOKEN and other private information, click the link to view specific steps.
+```
+
+## Start Runner
+
+add `/lib/systemd/system/runner.service`
+```
+[Unit]
+StartLimitIntervalSec=0
+[Service]
+Restart=always
+RestartSec=1
+ExecStart=/home/lmzheng/zhyncs/actions-runner/run.sh
+[Install]
+WantedBy=multi-user.target
+```
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl start runner
+sudo systemctl enable runner
+sudo systemctl status runner
+```
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sglang"
-version = "0.2.6"
+version = "0.2.7"
 description = "SGLang is yet another fast serving framework for large language models and vision language models."
 readme = "README.md"
 requires-python = ">=3.8"

diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py
@@ -1,5 +1,6 @@
 # Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py
 # Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py
+
 """
 Benchmark online serving.
 

diff --git a/python/sglang/srt/layers/radix_attention.py b/python/sglang/srt/layers/radix_attention.py
@@ -103,7 +103,7 @@ def decode_forward_triton(self, q, k, v, input_metadata: InputMetadata):
         return o
 
     def extend_forward_flashinfer(self, q, k, v, input_metadata: InputMetadata):
-        if not input_metadata.use_ragged:
+        if not input_metadata.flashinfer_use_ragged:
             self.store_kv_cache(k, v, input_metadata)
 
             o = input_metadata.flashinfer_prefill_wrapper_paged.forward(

diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py
@@ -781,7 +781,7 @@ class InputMetadata:
     flashinfer_prefill_wrapper_ragged: "BatchPrefillWithRaggedKVCacheWrapper" = None
     flashinfer_prefill_wrapper_paged: "BatchPrefillWithPagedKVCacheWrapper" = None
     flashinfer_decode_wrapper: "BatchDecodeWithPagedKVCacheWrapper" = None
-    use_ragged: bool = False
+    flashinfer_use_ragged: bool = False
 
     @classmethod
     def create(
@@ -797,18 +797,18 @@ def create(
         return_logprob=False,
         skip_flashinfer_init=False,
     ):
-        use_ragged = False
+        flashinfer_use_ragged = False
         if not skip_flashinfer_init and not model_runner.server_args.disable_flashinfer:
             if forward_mode != ForwardMode.DECODE and int(torch.sum(seq_lens)) > 4096:
-                use_ragged = True
+                flashinfer_use_ragged = True
             init_flashinfer_args(
                 forward_mode,
                 model_runner,
                 req_pool_indices,
                 seq_lens,
                 prefix_lens,
                 model_runner.flashinfer_decode_wrapper,
-                use_ragged,
+                flashinfer_use_ragged,
             )
 
         batch_size = len(req_pool_indices)
@@ -863,7 +863,7 @@ def create(
             flashinfer_prefill_wrapper_ragged=model_runner.flashinfer_prefill_wrapper_ragged,
             flashinfer_prefill_wrapper_paged=model_runner.flashinfer_prefill_wrapper_paged,
             flashinfer_decode_wrapper=model_runner.flashinfer_decode_wrapper,
-            use_ragged=use_ragged,
+            flashinfer_use_ragged=flashinfer_use_ragged,
         )
 
         if model_runner.server_args.disable_flashinfer:
@@ -884,7 +884,7 @@ def init_flashinfer_args(
     seq_lens,
     prefix_lens,
     flashinfer_decode_wrapper,
-    use_ragged=False,
+    flashinfer_use_ragged=False,
 ):
     """Init auxiliary variables for FlashInfer attention backend."""
     num_qo_heads = model_runner.model_config.num_attention_heads // model_runner.tp_size
@@ -893,7 +893,7 @@ def init_flashinfer_args(
     batch_size = len(req_pool_indices)
     total_num_tokens = int(torch.sum(seq_lens))
 
-    if use_ragged:
+    if flashinfer_use_ragged:
         paged_kernel_lens = prefix_lens
     else:
         paged_kernel_lens = seq_lens
@@ -929,7 +929,7 @@ def init_flashinfer_args(
         qo_indptr = torch.zeros((batch_size + 1,), dtype=torch.int32, device="cuda")
         qo_indptr[1:] = torch.cumsum(seq_lens - prefix_lens, dim=0)
 
-        if use_ragged:
+        if flashinfer_use_ragged:
             model_runner.flashinfer_prefill_wrapper_ragged.end_forward()
             model_runner.flashinfer_prefill_wrapper_ragged.begin_forward(
                 qo_indptr,

diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py
@@ -212,9 +212,14 @@ def init_memory_pool(self, total_gpu_memory, max_num_reqs=None):
             )
 
         if max_num_reqs is None:
-            max_num_reqs = max(
-                int(self.max_total_num_tokens / self.model_config.context_len * 512),
-                2048,
+            max_num_reqs = min(
+                max(
+                    int(
+                        self.max_total_num_tokens / self.model_config.context_len * 512
+                    ),
+                    2048,
+                ),
+                5120,
             )
 
         self.req_to_token_pool = ReqToTokenPool(

diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py
@@ -91,15 +91,15 @@ def __post_init__(self):
             self.tokenizer_path = self.model_path
         if self.mem_fraction_static is None:
             if self.tp_size >= 16:
-                self.mem_fraction_static = 0.80
+                self.mem_fraction_static = 0.79
             elif self.tp_size >= 8:
-                self.mem_fraction_static = 0.84
+                self.mem_fraction_static = 0.83
             elif self.tp_size >= 4:
-                self.mem_fraction_static = 0.86
+                self.mem_fraction_static = 0.85
             elif self.tp_size >= 2:
-                self.mem_fraction_static = 0.88
+                self.mem_fraction_static = 0.87
             else:
-                self.mem_fraction_static = 0.89
+                self.mem_fraction_static = 0.88
         if isinstance(self.additional_ports, int):
             self.additional_ports = [self.additional_ports]
         elif self.additional_ports is None:
@@ -195,6 +195,7 @@ def add_cli_args(parser: argparse.ArgumentParser):
                 "gptq",
                 "marlin",
                 "gptq_marlin",
+                "awq_marlin",
                 "squeezellm",
                 "bitsandbytes",
             ],

diff --git a/python/sglang/version.py b/python/sglang/version.py
@@ -1 +1 @@
-__version__ = "0.2.6"
+__version__ = "0.2.7"