[LLM] disable part of MC2 in lora

PaddlePaddle · May 29, 2024 · 24ac1d2 · 24ac1d2
1 parent c1cfe63
commit 24ac1d2
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/paddlenlp/peft/lora/lora_layers.py b/paddlenlp/peft/lora/lora_layers.py
@@ -399,7 +399,8 @@ def forward(self, x: paddle.Tensor):
         else:
             input_mp = x
 
-        if MC2RowSeqParallelCoreLinear is None:
+        # TODO(@gexiao): temporary workaround for deterministic calculation
+        if True or MC2RowSeqParallelCoreLinear is None:
             output_parallel = self.linear(input_mp, self.weight, name=self._name)
             output_ = ReduceScatterOp.apply(output_parallel)
             result_mp = output_ + self.bias if self.bias is not None else output_
@@ -651,7 +652,8 @@ def forward(self, x: paddle.Tensor):
 
         if not self.merged:
             input_a = self.lora_dropout(x) @ self.lora_A
-            if MC2ColumnSeqParallelCoreLinear is None:
+            # TODO(@gexiao): temporary workaround for deterministic calculation
+            if True or MC2ColumnSeqParallelCoreLinear is None:
                 input_a = AllGatherOp.apply(input_a)
                 delta_mp = (input_a @ self.lora_B) * self.scaling
             else: