update baichun2_13b config

PaddlePaddle · Jul 15, 2024 · 612a251 · 612a251
1 parent cf5818d
commit 612a251
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/...bf16_DP1_MP1_PP8_VPP1_Sharding4_Stage1.sh → ...bf16_DP1_MP2_PP2_VPP1_Sharding8_Stage1.sh b/...bf16_DP1_MP1_PP8_VPP1_Sharding4_Stage1.sh → ...bf16_DP1_MP2_PP2_VPP1_Sharding8_Stage1.sh
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 param="model_item=baichuan-inc-Baichun2-13b_pretrain "
-param+="run_mode=DP1_MP1_PP8_VPP1_Sharding4_Stage1 "
+param+="run_mode=DP1_MP2_PP2_VPP1_Sharding8_Stage1 "
 param+="device_num=N4C32 "
 param+="global_batch_size=32 "
 param+="nnodes=4 "

diff --git a/...ph/hybrid_parallelism/baichun2/auto_config_baichun2_13b/pretrain-baichun2_13b-config.json b/...ph/hybrid_parallelism/baichun2/auto_config_baichun2_13b/pretrain-baichun2_13b-config.json
@@ -5,12 +5,12 @@
   "output_dir": "./output/baichun2-13b_pretrain_ckpts",
   "split": "949,50,1",
   "max_seq_length": 4096,
-  "gradient_accumulation_steps": 8,
-  "tensor_parallel_degree": 1,
-  "pipeline_parallel_degree": 8,
+  "gradient_accumulation_steps": 4,
+  "tensor_parallel_degree": 2,
+  "pipeline_parallel_degree": 2,
   "virtual_pp_degree": 1,
-  "sequence_parallel": 0,   
-  "sharding_parallel_degree": 4,
+  "sequence_parallel": 1,   
+  "sharding_parallel_degree": 8,
   "sharding": "stage1",
   "pipeline_parallel_config": "enable_delay_scale_loss enable_sharding_comm_overlap enable_release_grads ",
   "tensor_parallel_config": "enable_delay_scale_loss enable_mp_async_allreduce enable_mp_skip_c_identity enable_mp_fused_linear_param_grad_add",