Minor improvements

Xia-Weiwen · May 10, 2024 · 177bd39 · 177bd39
1 parent 09cc153
commit 177bd39
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/bitsandbytes/backends/cpu.py b/bitsandbytes/backends/cpu.py
@@ -136,6 +136,7 @@ def quantize_4bit(
         quant_storage=torch.uint8,
     ) -> Tuple[torch.Tensor, QuantState]:
         assert_on_cpu([A, absmax, out])
+        assert quant_storage == torch.uint8, "CPU backend only supports uint8 quant_storage"
         return quantize_4bit_impl(A, absmax, out, blocksize, compress_statistics, quant_type)
 
     def dequantize_4bit(

diff --git a/bitsandbytes/backends/cpu_xpu_common.py b/bitsandbytes/backends/cpu_xpu_common.py
@@ -343,6 +343,8 @@ def quantize_4bit_impl(
         )
 
     if ipex_cpu and _ipex_cpu_version_prereq(2, 2) and input_shape[0] % blocksize == 0:
+        # lowp_mode: lowest precision for computation
+        lowp_mode = ipex_cpu.quantization.WoqLowpMode.BF16
         state.op_context = torch.ops.ipex_prepack.weight_only_qlinear_prepack(
             out.reshape([input_shape[0], input_shape[1] // 2]),
             ipex_cpu.quantization.WoqWeightDtype.NF4,
@@ -353,8 +355,8 @@ def quantize_4bit_impl(
             None, # g_idx
             None, # batch_size
             blocksize,
-            int(ipex_cpu.quantization.WoqLowpMode.BF16),
-            -1, # act_quant_mode
+            int(lowp_mode),
+            -1, # act_quant_mode. -1 means don't quant activation
         )
 
     return out, state