feat: Initial model params for granite code 3b

Branch: GraniteCodeSupport Signed-off-by: Gabe Goodhart <ghart@us.ibm.com>
gabe-l-hart · Oct 4, 2024 · cd2d131 · cd2d131
1 parent 82ee513
commit cd2d131
Showing 1 changed file with 16 additions and 0 deletions.
diff --git a/torchchat/model_params/Granite-3B-Code.json b/torchchat/model_params/Granite-3B-Code.json
@@ -0,0 +1,16 @@
+{
+    "block_size": 128000,
+    "dim": 2560,
+    "hidden_dim": 10240,
+    "n_heads": 32,
+    "n_local_heads": 32,
+    "n_layers": 32,
+    "rope_base": 10000000,
+    "vocab_size": 49152,
+    "use_tokenizers": true,
+    "norm_eps": 0.00001,
+    "rope_scaling": null,
+    "attention_bias": true,
+    "feed_forward_bias": true,
+    "tie_word_embeddings": true
+}