From 11d202846b8fb64f793548b97c08903b14e8e47d Mon Sep 17 00:00:00 2001
From: sedrick-keh-tri <133716510+sedrick-keh-tri@users.noreply.github.com>
Date: Fri, 19 Jul 2024 14:11:26 -0700
Subject: [PATCH] Create open_lm_1b_swiglutorch.json

---
 open_lm/model_configs/open_lm_1b_swiglutorch.json | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 open_lm/model_configs/open_lm_1b_swiglutorch.json

diff --git a/open_lm/model_configs/open_lm_1b_swiglutorch.json b/open_lm/model_configs/open_lm_1b_swiglutorch.json
new file mode 100644
index 00000000..d2f9b814
--- /dev/null
+++ b/open_lm/model_configs/open_lm_1b_swiglutorch.json
@@ -0,0 +1,10 @@
+{
+    "hidden_dim": 2048,
+    "n_layers": 24,
+    "n_heads": 16,
+    "seq_len": 2048,
+    "vocab_size": 50432,
+    "post_embed_norm": false,
+    "weight_tying": false,
+    "ffn_type": "swiglu_torch"
+}