add comments

open-compass · May 29, 2024 · 3801191 · 3801191
1 parent 79bdbc5
commit 3801191
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 4 deletions.
diff --git a/configs/models/mistral/vllm_mixtral_8x7b_instruct_v0_1.py b/configs/models/mistral/vllm_mixtral_8x7b_instruct_v0_1.py
@@ -8,16 +8,18 @@
         dict(role='BOT', begin='', end='</s>', generate=True),
     ],
 )
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='mixtral-8x7b-instruct-v0.1-vllm',
         path='mistralai/Mixtral-8x7B-Instruct-v0.1',
-        model_kwargs=dict(tensor_parallel_size=2),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=2, max_model_len=max_seq_len),
         meta_template=_meta_template,
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         stop_words=['</s>'],

diff --git a/configs/models/wizardlm/vllm_wizardlm_70b_v1_0.py b/configs/models/wizardlm/vllm_wizardlm_70b_v1_0.py
@@ -7,16 +7,18 @@
         dict(role='BOT', begin='ASSISTANT: ', end='</s>', generate=True),
     ],
 )
+max_seq_len = 2048
 
 models = [
     dict(
         type=VLLM,
         abbr='wizardlm-70b-v1.0-vllm',
         path='WizardLM/WizardLM-70B-V1.0',
-        model_kwargs=dict(tensor_parallel_size=4),
+        # more vllm model_kwargs: https://github.com/vllm-project/vllm/blob/main/vllm/engine/arg_utils.py
+        model_kwargs=dict(tensor_parallel_size=4, max_model_len=max_seq_len),
         meta_template=_meta_template,
         max_out_len=100,
-        max_seq_len=2048,
+        max_seq_len=max_seq_len,
         batch_size=32,
         generation_kwargs=dict(temperature=0),
         stop_words=['</s>'],