olmo2, tulu, granite (#26)

cohere-ai · Nov 28, 2024 · b905470 · b905470
1 parent d0ee01c
commit b905470
Show file tree

Hide file tree

Showing 100 changed files with 32,912 additions and 11 deletions.
diff --git a/magikarp/tokenization.py b/magikarp/tokenization.py
@@ -20,6 +20,7 @@ def model_needs_fast_tokenizer(model_id):
         or "OLMo" in model_id
         or "Mistral-Nemo" in model_id
         or "solar-pro-preview" in model_id
+        or 'ibm-granite/' in model_id
     )
 
 
@@ -55,7 +56,7 @@ def __init__(
             self.tokenizer.pad_token_id = eos_token_id
 
         self.model_id = model_id
-        self.vocab_s2i = self.tokenizer.get_vocab()
+        self.vocab_s2i = {k: v for k, v in self.tokenizer.get_vocab().items()}
         self.vocab_i2s = {v: k for k, v in self.vocab_s2i.items()}
 
         self.special_token_ids = getattr(self.tokenizer, "additional_special_tokens_ids", None) or []

diff --git a/magikarp/unused_tokens.py b/magikarp/unused_tokens.py
@@ -74,12 +74,17 @@
     "Qwen/Qwen1.5-32B": TIKTOKEN_UNUSED_TOKENS,
     "Qwen/Qwen2-57B-A14B": TIKTOKEN_UNUSED_TOKENS,
     "Qwen/Qwen2.5-32B-Instruct": TIKTOKEN_UNUSED_TOKENS,
+    "Qwen/Qwen2.5-7B": TIKTOKEN_UNUSED_TOKENS,
+    "Qwen/Qwen2.5-7B-Instruct": TIKTOKEN_UNUSED_TOKENS,
     "stabilityai/stablelm-2-12b": TIKTOKEN_UNUSED_TOKENS,
     "meta-llama/Meta-Llama-3-8B": TIKTOKEN_UNUSED_TOKENS,
     "meta-llama/Meta-Llama-3.1-8B": TIKTOKEN_UNUSED_TOKENS,
     "meta-llama/Meta-Llama-3-70B": TIKTOKEN_UNUSED_TOKENS,
     "meta-llama/Meta-Llama-3.1-70B": TIKTOKEN_UNUSED_TOKENS,
     "internlm/internlm2_5-7b-chat": TIKTOKEN_UNUSED_TOKENS,
+    "allenai/Llama-3.1-Tulu-3-8B": TIKTOKEN_UNUSED_TOKENS,
+    "allenai/OLMo-2-1124-7B": TIKTOKEN_UNUSED_TOKENS,
+    "allenai/OLMo-2-1124-13B": TIKTOKEN_UNUSED_TOKENS,
     "mistralai/Mistral-Nemo-Base-2407": np.arange(1245, 1256),  # f5-ff
     # deepseek
     "deepseek-ai/deepseek-llm-7b-base": DEEPSEEK_LANG_UNUSED_TOKENS,
@@ -101,4 +106,5 @@
     "m-a-p/neo_7b": MAP_NEO_UNUSED_TOKENS,
     "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct": np.arange(100, 150),  # unused ...
     "utter-project/EuroLLM-1.7B": np.arange(5, 260),  # extra
+    "ibm-granite/granite-3.0-8b-base": STARCODER2_UNUSED_TOKENS,
 }
diff --git a/poetry.lock b/poetry.lock
diff --git a/results/indicators_pairplot_byid/Qwen_Qwen2_5_7B.png b/results/indicators_pairplot_byid/Qwen_Qwen2_5_7B.png
diff --git a/results/indicators_pairplot_byid/Qwen_Qwen2_5_7B_Instruct.png b/results/indicators_pairplot_byid/Qwen_Qwen2_5_7B_Instruct.png
diff --git a/results/indicators_pairplot_byid/allenai_Llama_3_1_Tulu_3_8B.png b/results/indicators_pairplot_byid/allenai_Llama_3_1_Tulu_3_8B.png
diff --git a/results/indicators_pairplot_byid/allenai_OLMo_2_1124_13B.png b/results/indicators_pairplot_byid/allenai_OLMo_2_1124_13B.png
diff --git a/results/indicators_pairplot_byid/allenai_OLMo_2_1124_7B.png b/results/indicators_pairplot_byid/allenai_OLMo_2_1124_7B.png
diff --git a/results/indicators_pairplot_byid/ibm_granite_granite_3_0_8b_base.png b/results/indicators_pairplot_byid/ibm_granite_granite_3_0_8b_base.png
diff --git a/results/reports/Qwen_Qwen2_5_7B.md b/results/reports/Qwen_Qwen2_5_7B.md
diff --git a/results/reports/Qwen_Qwen2_5_7B_Instruct.md b/results/reports/Qwen_Qwen2_5_7B_Instruct.md
diff --git a/results/reports/allenai_Llama_3_1_Tulu_3_8B.md b/results/reports/allenai_Llama_3_1_Tulu_3_8B.md
diff --git a/results/reports/allenai_OLMo_2_1124_13B.md b/results/reports/allenai_OLMo_2_1124_13B.md
diff --git a/results/reports/allenai_OLMo_2_1124_7B.md b/results/reports/allenai_OLMo_2_1124_7B.md
diff --git a/results/reports/ibm_granite_granite_3_0_8b_base.md b/results/reports/ibm_granite_granite_3_0_8b_base.md
diff --git a/results/reports_mini/Qwen_Qwen2_5_7B.md b/results/reports_mini/Qwen_Qwen2_5_7B.md
diff --git a/results/reports_mini/Qwen_Qwen2_5_7B_Instruct.md b/results/reports_mini/Qwen_Qwen2_5_7B_Instruct.md
diff --git a/results/reports_mini/allenai_Llama_3_1_Tulu_3_8B.md b/results/reports_mini/allenai_Llama_3_1_Tulu_3_8B.md
diff --git a/results/reports_mini/allenai_OLMo_2_1124_13B.md b/results/reports_mini/allenai_OLMo_2_1124_13B.md
diff --git a/results/reports_mini/allenai_OLMo_2_1124_7B.md b/results/reports_mini/allenai_OLMo_2_1124_7B.md
diff --git a/results/reports_mini/ibm_granite_granite_3_0_8b_base.md b/results/reports_mini/ibm_granite_granite_3_0_8b_base.md
diff --git a/results/summary.md b/results/summary.md
diff --git a/results/verifications/01_ai_Yi_1_5_9B.jsonl.gz b/results/verifications/01_ai_Yi_1_5_9B.jsonl.gz
diff --git a/results/verifications/01_ai_Yi_9B.jsonl.gz b/results/verifications/01_ai_Yi_9B.jsonl.gz
diff --git a/results/verifications/01_ai_Yi_Coder_9B_Chat.jsonl.gz b/results/verifications/01_ai_Yi_Coder_9B_Chat.jsonl.gz
diff --git a/results/verifications/152334H_miqu_1_70b_sf.jsonl.gz b/results/verifications/152334H_miqu_1_70b_sf.jsonl.gz
diff --git a/results/verifications/CohereForAI_aya_23_35B.jsonl.gz b/results/verifications/CohereForAI_aya_23_35B.jsonl.gz
diff --git a/results/verifications/CohereForAI_c4ai_command_r_plus.jsonl.gz b/results/verifications/CohereForAI_c4ai_command_r_plus.jsonl.gz
diff --git a/results/verifications/CohereForAI_c4ai_command_r_v01.jsonl.gz b/results/verifications/CohereForAI_c4ai_command_r_v01.jsonl.gz
diff --git a/results/verifications/EleutherAI_gpt_j_6b.jsonl.gz b/results/verifications/EleutherAI_gpt_j_6b.jsonl.gz
diff --git a/results/verifications/EleutherAI_gpt_neox_20b.jsonl.gz b/results/verifications/EleutherAI_gpt_neox_20b.jsonl.gz
diff --git a/results/verifications/EleutherAI_pythia_6_9b.jsonl.gz b/results/verifications/EleutherAI_pythia_6_9b.jsonl.gz
diff --git a/results/verifications/Fugaku_LLM_Fugaku_LLM_13B.jsonl.gz b/results/verifications/Fugaku_LLM_Fugaku_LLM_13B.jsonl.gz
diff --git a/results/verifications/HuggingFaceH4_zephyr_7b_beta.jsonl.gz b/results/verifications/HuggingFaceH4_zephyr_7b_beta.jsonl.gz
diff --git a/results/verifications/LGAI_EXAONE_EXAONE_3_0_7_8B_Instruct.jsonl.gz b/results/verifications/LGAI_EXAONE_EXAONE_3_0_7_8B_Instruct.jsonl.gz
diff --git a/results/verifications/Nexusflow_Starling_LM_7B_beta.jsonl.gz b/results/verifications/Nexusflow_Starling_LM_7B_beta.jsonl.gz
diff --git a/results/verifications/Qwen_Qwen1_5_32B.jsonl.gz b/results/verifications/Qwen_Qwen1_5_32B.jsonl.gz
diff --git a/results/verifications/Qwen_Qwen1_5_72B_Chat.jsonl.gz b/results/verifications/Qwen_Qwen1_5_72B_Chat.jsonl.gz
diff --git a/results/verifications/Qwen_Qwen1_5_MoE_A2_7B.jsonl.gz b/results/verifications/Qwen_Qwen1_5_MoE_A2_7B.jsonl.gz
diff --git a/results/verifications/Qwen_Qwen2_57B_A14B.jsonl.gz b/results/verifications/Qwen_Qwen2_57B_A14B.jsonl.gz
diff --git a/results/verifications/Qwen_Qwen2_5_32B_Instruct.jsonl.gz b/results/verifications/Qwen_Qwen2_5_32B_Instruct.jsonl.gz
diff --git a/results/verifications/Qwen_Qwen2_5_7B.jsonl.gz b/results/verifications/Qwen_Qwen2_5_7B.jsonl.gz
diff --git a/results/verifications/Qwen_Qwen2_5_7B_Instruct.jsonl.gz b/results/verifications/Qwen_Qwen2_5_7B_Instruct.jsonl.gz
diff --git a/results/verifications/Rakuten_RakutenAI_7B.jsonl.gz b/results/verifications/Rakuten_RakutenAI_7B.jsonl.gz
diff --git a/results/verifications/Skywork_Skywork_13B_base.jsonl.gz b/results/verifications/Skywork_Skywork_13B_base.jsonl.gz
diff --git a/results/verifications/ai21labs_AI21_Jamba_1_5_Mini.jsonl.gz b/results/verifications/ai21labs_AI21_Jamba_1_5_Mini.jsonl.gz
diff --git a/results/verifications/ai21labs_Jamba_v0_1.jsonl.gz b/results/verifications/ai21labs_Jamba_v0_1.jsonl.gz
diff --git a/results/verifications/allenai_Llama_3_1_Tulu_3_8B.jsonl.gz b/results/verifications/allenai_Llama_3_1_Tulu_3_8B.jsonl.gz
diff --git a/results/verifications/allenai_OLMoE_1B_7B_0924.jsonl.gz b/results/verifications/allenai_OLMoE_1B_7B_0924.jsonl.gz
diff --git a/results/verifications/allenai_OLMo_1_7_7B_hf.jsonl.gz b/results/verifications/allenai_OLMo_1_7_7B_hf.jsonl.gz
diff --git a/results/verifications/allenai_OLMo_2_1124_13B.jsonl.gz b/results/verifications/allenai_OLMo_2_1124_13B.jsonl.gz
diff --git a/results/verifications/allenai_OLMo_2_1124_7B.jsonl.gz b/results/verifications/allenai_OLMo_2_1124_7B.jsonl.gz
diff --git a/results/verifications/allenai_OLMo_7B_hf.jsonl.gz b/results/verifications/allenai_OLMo_7B_hf.jsonl.gz
diff --git a/results/verifications/benjamin_Mistral_7B_v0_1_zett_gpt2.jsonl.gz b/results/verifications/benjamin_Mistral_7B_v0_1_zett_gpt2.jsonl.gz
diff --git a/results/verifications/bigcode_starcoder2_15b.jsonl.gz b/results/verifications/bigcode_starcoder2_15b.jsonl.gz
diff --git a/results/verifications/chuxin_llm_Chuxin_1_6B_Base.jsonl.gz b/results/verifications/chuxin_llm_Chuxin_1_6B_Base.jsonl.gz
diff --git a/results/verifications/deepseek_ai_DeepSeek_V2_Lite.jsonl.gz b/results/verifications/deepseek_ai_DeepSeek_V2_Lite.jsonl.gz
diff --git a/results/verifications/deepseek_ai_deepseek_coder_33b_base.jsonl.gz b/results/verifications/deepseek_ai_deepseek_coder_33b_base.jsonl.gz
diff --git a/results/verifications/deepseek_ai_deepseek_llm_7b_base.jsonl.gz b/results/verifications/deepseek_ai_deepseek_llm_7b_base.jsonl.gz
diff --git a/results/verifications/deepseek_ai_deepseek_math_7b_base.jsonl.gz b/results/verifications/deepseek_ai_deepseek_math_7b_base.jsonl.gz
diff --git a/results/verifications/facebook_xglm_7_5B.jsonl.gz b/results/verifications/facebook_xglm_7_5B.jsonl.gz
diff --git a/results/verifications/google_codegemma_7b.jsonl.gz b/results/verifications/google_codegemma_7b.jsonl.gz
diff --git a/results/verifications/google_gemma_2b.jsonl.gz b/results/verifications/google_gemma_2b.jsonl.gz
diff --git a/results/verifications/google_gemma_7b.jsonl.gz b/results/verifications/google_gemma_7b.jsonl.gz
diff --git a/results/verifications/h2oai_h2o_danube2_1_8b_base.jsonl.gz b/results/verifications/h2oai_h2o_danube2_1_8b_base.jsonl.gz
diff --git a/results/verifications/ibm_granite_granite_3_0_8b_base.jsonl.gz b/results/verifications/ibm_granite_granite_3_0_8b_base.jsonl.gz
diff --git a/results/verifications/internlm_internlm2_5_7b_chat.jsonl.gz b/results/verifications/internlm_internlm2_5_7b_chat.jsonl.gz
diff --git a/results/verifications/m_a_p_neo_7b.jsonl.gz b/results/verifications/m_a_p_neo_7b.jsonl.gz
diff --git a/results/verifications/meta_llama_Llama_2_13b_hf.jsonl.gz b/results/verifications/meta_llama_Llama_2_13b_hf.jsonl.gz
diff --git a/results/verifications/meta_llama_Llama_2_70b_hf.jsonl.gz b/results/verifications/meta_llama_Llama_2_70b_hf.jsonl.gz
diff --git a/results/verifications/meta_llama_Llama_2_7b_hf.jsonl.gz b/results/verifications/meta_llama_Llama_2_7b_hf.jsonl.gz
diff --git a/results/verifications/meta_llama_Meta_Llama_3_1_70B.jsonl.gz b/results/verifications/meta_llama_Meta_Llama_3_1_70B.jsonl.gz
diff --git a/results/verifications/meta_llama_Meta_Llama_3_1_8B.jsonl.gz b/results/verifications/meta_llama_Meta_Llama_3_1_8B.jsonl.gz
diff --git a/results/verifications/meta_llama_Meta_Llama_3_70B.jsonl.gz b/results/verifications/meta_llama_Meta_Llama_3_70B.jsonl.gz
diff --git a/results/verifications/meta_llama_Meta_Llama_3_8B.jsonl.gz b/results/verifications/meta_llama_Meta_Llama_3_8B.jsonl.gz
diff --git a/results/verifications/microsoft_Phi_3_5_mini_instruct.jsonl.gz b/results/verifications/microsoft_Phi_3_5_mini_instruct.jsonl.gz
diff --git a/results/verifications/microsoft_Phi_3_mini_128k_instruct.jsonl.gz b/results/verifications/microsoft_Phi_3_mini_128k_instruct.jsonl.gz
diff --git a/results/verifications/microsoft_phi_2.jsonl.gz b/results/verifications/microsoft_phi_2.jsonl.gz
diff --git a/results/verifications/mistralai_Codestral_22B_v0_1.jsonl.gz b/results/verifications/mistralai_Codestral_22B_v0_1.jsonl.gz
diff --git a/results/verifications/mistralai_Mistral_7B_Instruct_v0_2.jsonl.gz b/results/verifications/mistralai_Mistral_7B_Instruct_v0_2.jsonl.gz
diff --git a/results/verifications/mistralai_Mistral_7B_v0_1.jsonl.gz b/results/verifications/mistralai_Mistral_7B_v0_1.jsonl.gz
diff --git a/results/verifications/mistralai_Mistral_7B_v0_3.jsonl.gz b/results/verifications/mistralai_Mistral_7B_v0_3.jsonl.gz
diff --git a/results/verifications/mistralai_Mistral_Nemo_Base_2407.jsonl.gz b/results/verifications/mistralai_Mistral_Nemo_Base_2407.jsonl.gz
diff --git a/results/verifications/mistralai_Mixtral_8x7B_v0_1.jsonl.gz b/results/verifications/mistralai_Mixtral_8x7B_v0_1.jsonl.gz
diff --git a/results/verifications/nvidia_NVLM_D_72B.jsonl.gz b/results/verifications/nvidia_NVLM_D_72B.jsonl.gz
diff --git a/results/verifications/openai_community_gpt2.jsonl.gz b/results/verifications/openai_community_gpt2.jsonl.gz
diff --git a/results/verifications/openai_community_gpt2_large.jsonl.gz b/results/verifications/openai_community_gpt2_large.jsonl.gz
diff --git a/results/verifications/openai_community_gpt2_medium.jsonl.gz b/results/verifications/openai_community_gpt2_medium.jsonl.gz
diff --git a/results/verifications/openai_community_gpt2_xl.jsonl.gz b/results/verifications/openai_community_gpt2_xl.jsonl.gz
diff --git a/results/verifications/stabilityai_stablelm_2_12b.jsonl.gz b/results/verifications/stabilityai_stablelm_2_12b.jsonl.gz
diff --git a/results/verifications/upstage_SOLAR_10_7B_v1_0.jsonl.gz b/results/verifications/upstage_SOLAR_10_7B_v1_0.jsonl.gz
diff --git a/results/verifications/upstage_solar_pro_preview_instruct.jsonl.gz b/results/verifications/upstage_solar_pro_preview_instruct.jsonl.gz
diff --git a/results/verifications/utter_project_EuroLLM_1_7B.jsonl.gz b/results/verifications/utter_project_EuroLLM_1_7B.jsonl.gz
diff --git a/results/verifications_scatterplot/Qwen_Qwen2_5_7B.png b/results/verifications_scatterplot/Qwen_Qwen2_5_7B.png
diff --git a/results/verifications_scatterplot/Qwen_Qwen2_5_7B_Instruct.png b/results/verifications_scatterplot/Qwen_Qwen2_5_7B_Instruct.png
diff --git a/results/verifications_scatterplot/allenai_Llama_3_1_Tulu_3_8B.png b/results/verifications_scatterplot/allenai_Llama_3_1_Tulu_3_8B.png
diff --git a/results/verifications_scatterplot/allenai_OLMo_2_1124_13B.png b/results/verifications_scatterplot/allenai_OLMo_2_1124_13B.png
diff --git a/results/verifications_scatterplot/allenai_OLMo_2_1124_7B.png b/results/verifications_scatterplot/allenai_OLMo_2_1124_7B.png
diff --git a/results/verifications_scatterplot/ibm_granite_granite_3_0_8b_base.png b/results/verifications_scatterplot/ibm_granite_granite_3_0_8b_base.png
diff --git a/run_verification.sh b/run_verification.sh
@@ -67,11 +67,17 @@ for arg in "$@"; do
             python magikarp/fishing.py --model_id "Qwen/Qwen1.5-72B-Chat"
             python magikarp/fishing.py --model_id "Qwen/Qwen2-57B-A14B"
             python magikarp/fishing.py --model_id "Qwen/Qwen2.5-32B-Instruct"
+            python magikarp/fishing.py --model_id "Qwen/Qwen2.5-7B"
             ;;
         "tiktoken")
             python magikarp/fishing.py --model_id "stabilityai/stablelm-2-12b" --trust-remote-code # missing weights if not trust remote
             python magikarp/fishing.py --model_id "mistralai/Mistral-Nemo-Base-2407" --threshold_ratio 5
             ;;
+        "allenai")
+            python magikarp/fishing.py --model_id "allenai/Llama-3.1-Tulu-3-8B"
+            python magikarp/fishing.py --model_id  "allenai/OLMo-2-1124-7B" --indicator_ix 1
+            python magikarp/fishing.py --model_id  "allenai/OLMo-2-1124-13B" --indicator_ix 1
+            ;;
         "deepseek")
             python magikarp/fishing.py --model_id  "deepseek-ai/deepseek-llm-7b-base"
             python magikarp/fishing.py --model_id  "deepseek-ai/deepseek-math-7b-base"
@@ -94,6 +100,7 @@ for arg in "$@"; do
             ;;
         "misc")
             python magikarp/fishing.py --model_id "bigcode/starcoder2-15b"
+            python magikarp/fishing.py --model_id "ibm-granite/granite-3.0-8b-base"
             python magikarp/fishing.py --model_id "m-a-p/neo_7b" --trust-remote-code
             python magikarp/fishing.py --model_id "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct" --trust-remote-code
             python magikarp/fishing.py --model_id "internlm/internlm2_5-7b-chat" --trust-remote-code