need additional_special_tokens argument for HFLM initializer

Stability-AI · Oct 21, 2023 · 9a1e5b8 · 9a1e5b8
1 parent 5e80a0b
commit 9a1e5b8
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/lm_eval/models/gpt2.py b/lm_eval/models/gpt2.py
@@ -20,6 +20,7 @@ def __init__(
         load_in_8bit: Optional[bool] = False,
         trust_remote_code: Optional[bool] = False,
         use_fast: Optional[bool] = True,
+        additional_special_tokens: Optional[str] = None
     ):
         super().__init__()
 
@@ -66,6 +67,7 @@ def __init__(
             revision=revision,
             trust_remote_code=trust_remote_code,
             use_fast=use_fast,
+            additional_special_tokens=additional_special_tokens
         )
         self.vocab_size = self.tokenizer.vocab_size
 

diff --git a/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa-1.2.sh b/models/stabilityai/stabilityai-japanese-stablelm-base-alpha-7b/harness.jcqa-1.2.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 set -eu
-MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-base-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,offload_folder=/tmp"
+MODEL_ARGS="pretrained=stabilityai/japanese-stablelm-base-alpha-7b,use_fast=False,trust_remote_code=True,device_map=auto,torch_dtype=auto,offload_folder=/tmp,tokenizer=novelai/nerdstash-tokenizer-v1,additional_special_tokens=['▁▁']"
 TASK="jcommonsenseqa-1.2-0.2"
 NUM_FEW_SHOTS="3"
 python main.py \