Create open_lm_1b_swiglutorch.json

mlfoundations · Jul 19, 2024 · 11d2028 · 11d2028
1 parent 87bbb85
commit 11d2028
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/open_lm/model_configs/open_lm_1b_swiglutorch.json b/open_lm/model_configs/open_lm_1b_swiglutorch.json
@@ -0,0 +1,10 @@
+{
+    "hidden_dim": 2048,
+    "n_layers": 24,
+    "n_heads": 16,
+    "seq_len": 2048,
+    "vocab_size": 50432,
+    "post_embed_norm": false,
+    "weight_tying": false,
+    "ffn_type": "swiglu_torch"
+}