Message formatting

arcee-ai · Jan 3, 2024 · 4c99184 · 4c99184
1 parent 075333f
commit 4c99184
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/mergekit/scripts/mixtral_moe.py b/mergekit/scripts/mixtral_moe.py
@@ -175,6 +175,9 @@ def warn_degenerate_gates(gate_vecs: torch.Tensor, threshold: float = 5.0):
         elif len(degen_indices) == 2:
             layer_str = f"layers {' and '.join(map(str, degen_indices))}"
             verb = "have"
+        elif len(degen_indices) == num_layers:
+            layer_str = "ALL layers"
+            verb = "have"
         else:
             layer_str = (
                 "layers "