Add EXL2 Quantization

.gitignore

@@ -1,4 +1,7 @@
 /target
 .ruff_cache
 .vscode
-*.a
+*.a
+.DS_Store
+architecture.md
+.gitignore

Cargo.toml

@@ -4,12 +4,13 @@ members = [
     "mistralrs-core",
     "mistralrs-pyo3",
     "mistralrs",
-    "mistralrs-bench",
+    #"mistralrs-bench",
     "mistralrs-vision",
     "mistralrs-quant",
 ]
 exclude = [
     "mistralrs-paged_attn",
+    "mistralrs-bench",
 ]
 resolver = "2"
 
@@ -25,8 +26,8 @@ license = "MIT"
 
 [workspace.dependencies]
 anyhow = "1.0.80"
-candle-core = { git = "https://github.com/EricLBuehler/candle.git", version = "0.6.0", rev = "8a99f7c" }
-candle-nn = { git = "https://github.com/EricLBuehler/candle.git", version = "0.6.0", rev = "8a99f7c" }
+candle-core = { git = "https://github.com/EricLBuehler/candle.git", version = "0.6.0", rev = "61d19cc" }
+candle-nn = { git = "https://github.com/EricLBuehler/candle.git", version = "0.6.0", rev = "61d19cc" }
 serde = "1.0.197"
 serde_json = "1.0.114"
 indexmap = { version = "2.2.5", features = ["serde"] }

mistralrs-core/Cargo.toml

@@ -17,7 +17,7 @@ candle-core.workspace = true
 candle-nn.workspace = true
 serde.workspace = true
 serde_json.workspace = true
-candle-flash-attn = { git = "https://github.com/EricLBuehler/candle.git", version = "0.6.0", rev = "8a99f7c", optional = true }
+candle-flash-attn = { git = "https://github.com/EricLBuehler/candle.git", version = "0.6.0", rev = "61d19cc", optional = true }
 dirs = "5.0.1"
 hf-hub = "0.3.2"
 thiserror = "1.0.57"

mistralrs-core/build.rs

@@ -28,6 +28,7 @@ fn main() {
         // https://github.com/EricLBuehler/mistral.rs/issues/286
         if let Some(cuda_nvcc_flags_env) = CUDA_NVCC_FLAGS {
             builder = builder.arg("--compiler-options");
+            //builder = builder.arg("-fPIC -fPIE");
             builder = builder.arg(cuda_nvcc_flags_env);
         }