enable_sp_async_reduce_scatter for qwen_72b && llama2_70b

PaddlePaddle · Aug 8, 2024 · 86e8cf0 · 86e8cf0
1 parent 5c57015
commit 86e8cf0
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/...raph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json b/...raph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json
@@ -10,7 +10,7 @@
     "pipeline_parallel_degree": 8,
     "sharding": "stage1",
     "sharding_parallel_config": "split_param enable_stage1_overlap",
-    "tensor_parallel_config": "enable_delay_scale_loss enable_mp_async_allreduce enable_mp_skip_c_identity enable_mp_fused_linear_param_grad_add",
+    "tensor_parallel_config": "enable_delay_scale_loss enable_mp_async_allreduce enable_sp_async_reduce_scatter enable_mp_skip_c_identity enable_mp_fused_linear_param_grad_add",
     "pipeline_parallel_config": "enable_delay_scale_loss enable_release_grads disable_partial_send_recv enable_overlap_p2p_comm",
     "virtual_pp_degree": 5,
     "sequence_parallel": 1,   

diff --git a/...sm/qwen/N4C32/qwen-qwen-72b_seqlen4096_pretrain_bs32_bf16_MP8-PP4-sharding1-mbs1-acc32.sh b/...sm/qwen/N4C32/qwen-qwen-72b_seqlen4096_pretrain_bs32_bf16_MP8-PP4-sharding1-mbs1-acc32.sh
@@ -31,7 +31,7 @@ param+="model_item=qwen-qwen-72b_seqlen4096_pretrain "
 param+="max_steps=100 "
 param+="gradient_accumulation_steps=32 "
 param+="pp_recompute_interval=1 "
-param+="tensor_parallel_config=enable_delay_scale_loss,enable_mp_async_allreduce,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add, "
+param+="tensor_parallel_config=enable_delay_scale_loss,enable_mp_async_allreduce,enable_sp_async_reduce_scatter,enable_mp_skip_c_identity,enable_mp_fused_linear_param_grad_add, "
 #多机新添加的参数
 param+="pipeline_parallel_config=enable_delay_scale_loss,enable_sharding_comm_overlap,enable_release_grads, "
 param+="max_seq_length=4096 "