config adjustments for llama and gated activations

EleutherAI · Oct 25, 2024 · dc4b81f · dc4b81f
1 parent 575c4b6
commit dc4b81f
Show file tree

Hide file tree

Showing 6 changed files with 7 additions and 3 deletions.
diff --git a/configs/llama/13B.yml b/configs/llama/13B.yml
@@ -17,6 +17,7 @@
   "output_layer_parallelism": "column",
   "norm": "rmsnorm",
   "rms_norm_epsilon": 1.0e-6,
+  "use_bias_in_mlp": False,
 
   "scaled_upper_triang_masked_softmax_fusion": true,
   "bias_gelu_fusion": false,

diff --git a/configs/llama/30B.yml b/configs/llama/30B.yml
@@ -17,6 +17,7 @@
   "output_layer_parallelism": "column",
   "norm": "rmsnorm",
   "rms_norm_epsilon": 1.0e-6,
+  "use_bias_in_mlp": False,
 
   "scaled_upper_triang_masked_softmax_fusion": true,
   "bias_gelu_fusion": false,

diff --git a/configs/llama/65B.yml b/configs/llama/65B.yml
@@ -17,6 +17,7 @@
   "output_layer_parallelism": "column",
   "norm": "rmsnorm",
   "rms_norm_epsilon": 1.0e-6,
+  "use_bias_in_mlp": False,
 
   "scaled_upper_triang_masked_softmax_fusion": true,
   "bias_gelu_fusion": false,

diff --git a/configs/llama/7B.yml b/configs/llama/7B.yml
@@ -17,6 +17,7 @@
   "output_layer_parallelism": "column",
   "norm": "rmsnorm",
   "rms_norm_epsilon": 1.0e-6,
+  "use_bias_in_mlp": False,
 
   "scaled_upper_triang_masked_softmax_fusion": true,
   "bias_gelu_fusion": false,

diff --git a/configs/llama/train_config.yml b/configs/llama/train_config.yml
@@ -70,5 +70,5 @@
   "steps_per_print": 10,
   "keep_last_n_checkpoints": 4,
   "wall_clock_breakdown": true,
-  "mlp_multiple_of": 256,
+
 }
diff --git a/megatron/model/transformer.py b/megatron/model/transformer.py
@@ -1269,8 +1269,8 @@ def forward(self, x, attention_mask, layer_past=None):
 
             with torch.enable_grad() if not self.eval else nullcontext():
                 if (
-                    self.activation == "swiglu"
-                    or self.num_experts > 1
+                    mlp_bias == None,
+                    self.num_experts > 1
                     and self.moe_type == "deepspeed"
                 ):
                     # No dropout either