fix

PaddlePaddle · Jan 23, 2025 · a2ac30d · a2ac30d
1 parent ff86075
commit a2ac30d
Show file tree

Hide file tree

Showing 5 changed files with 146 additions and 193 deletions.
diff --git a/paddlenlp/mergekit/merge_config.py b/paddlenlp/mergekit/merge_config.py
@@ -17,10 +17,7 @@
 from dataclasses import asdict, dataclass, field
 from typing import List, Optional
 
-import paddle
-
 from paddlenlp.utils.env import MERGE_CONFIG_NAME
-from paddlenlp.utils.log import logger
 
 
 @dataclass
@@ -30,7 +27,6 @@ class MergeConfig:
     """
 
     # Common parameters
-    device: str = field(default="cpu", metadata={"help": "Device to use for the merge.ex cpu、 gpu、low_gpu_mem"})
     tensor_type: str = field(
         default="np", metadata={"help": "Tensor type to use for the merge. Choose np(CPU Only) or pd (CPU/GPU)"}
     )
@@ -47,6 +43,8 @@ class MergeConfig:
     )
     base_model_path: str = field(default=None, metadata={"help": "Base model name or path."})
     output_path: str = field(default=None, metadata={"help": "Base model name or path."})
+    lora_merge: bool = field(default=False, metadata={"help": "Use LoRA merge."})
+
     # merge parameters
     weight_list: Optional[List[float]] = field(
         default=None, metadata={"help": "Relative (or absolute if normalize=False) weighting of a given tensor"}
@@ -75,9 +73,6 @@ def config_check(self):
             os.makedirs(self.output_path, exist_ok=True)
         if self.tensor_type not in ["np", "pd"]:
             raise ValueError(f"Unsupported tensor type: {self.tensor_type}. Support 'np' and 'pd' only.")
-        if self.device == "gpu" and self.tensor_type == "np":
-            logger.warning("np only support cpu device, but got gpu. Setting `device` to `cpu`.")
-            self.device = "cpu"
 
         elif self.merge_method not in ["linear", "ties", "slerp", "della_linear", "della", "dare_linear", "dare_ties"]:
             raise ValueError(
@@ -100,7 +95,6 @@ def config_check(self):
                 raise ValueError(
                     f"Error: reserve_p +- epsilon/2 must be in the range (0, 1). reserve_p + epsilon/2 = {self.reserve_p + self.epsilon / 2 }, reserve_p - epsilon/2 = {self.reserve_p - self.epsilon / 2 }"
                 )
-        paddle.set_device(self.device)
 
     @property
     def __dict__(self):

diff --git a/paddlenlp/mergekit/merge_method.py b/paddlenlp/mergekit/merge_method.py
@@ -48,11 +48,10 @@ def linear(self, tensor_list):
             tensor_output = sum(weight * tensor for weight, tensor in zip(weight_list, tensor_list))
             return tensor_output
         elif self.merge_config.tensor_type == "pd":
-            weighted_sum = paddle.zeros_like(tensor_list[0])
+            tensor_output = paddle.zeros_like(tensor_list[0])
             for i, tensor in enumerate(tensor_list):
-                weight = weight_list[i]
-                weighted_sum += tensor * weight
-            return weighted_sum
+                tensor_output += tensor * weight_list[i]
+            return tensor_output
         else:
             raise ValueError(f"Unkonwn tensor type {self.merge_config.tensor_type}")