diff --git a/intel_extension_for_transformers/llm/quantization/nn/cpu/modules.py b/intel_extension_for_transformers/llm/quantization/nn/cpu/modules.py
index 79781b985e8..bffece572ad 100644
--- a/intel_extension_for_transformers/llm/quantization/nn/cpu/modules.py
+++ b/intel_extension_for_transformers/llm/quantization/nn/cpu/modules.py
@@ -25,7 +25,7 @@
 
 
 torch.ops.load_library(
-    os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../..", "libqbits.so")
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../../..", "libqbits.so")
 )
 
 
diff --git a/intel_extension_for_transformers/llm/quantization/nn/gpu/modules.py b/intel_extension_for_transformers/llm/quantization/nn/gpu/modules.py
index 412c93f9576..070e2d1e438 100644
--- a/intel_extension_for_transformers/llm/quantization/nn/gpu/modules.py
+++ b/intel_extension_for_transformers/llm/quantization/nn/gpu/modules.py
@@ -25,7 +25,7 @@
 
 
 torch.ops.load_library(
-    os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../..", "libqbits.so")
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../../..", "libqbits.so")
 )