Support disable_ignore_eos in bench_serving.py (sgl-project#824)

haichuan1221 · Jul 30, 2024 · ae5c0fc · ae5c0fc
1 parent a30d5d7
commit ae5c0fc
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py
@@ -84,6 +84,9 @@ async def async_request_trt_llm(
             "min_length": request_func_input.output_len,
             "end_id": 1048576,
         }
+        if args.disable_ignore_eos:
+            del payload["min_length"]
+            del payload["end_id"]
         output = RequestFuncOutput()
         output.prompt_len = request_func_input.prompt_len
 
@@ -149,7 +152,7 @@ async def async_request_openai_completions(
             "best_of": 1,
             "max_tokens": request_func_input.output_len,
             "stream": not args.disable_stream,
-            "ignore_eos": True,
+            "ignore_eos": not args.disable_ignore_eos,
         }
         headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
 
@@ -969,6 +972,11 @@ def set_ulimit(target_soft_limit=65535):
         action="store_true",
         help="Disable streaming mode.",
     )
+    parser.add_argument(
+        "--disable-ignore-eos",
+        action="store_true",
+        help="Disable ignoring EOS.",
+    )
 
     set_ulimit()