step

huggingface · Jun 13, 2024 · 447dae6 · 447dae6
1 parent f33f949
commit 447dae6
Showing 1 changed file with 1 addition and 4 deletions.
diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py
@@ -1958,9 +1958,6 @@ def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False)
             # We also expose some options like custom indents and separators
             return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys)
 
-        jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)
-        jinja_env.filters["tojson"] = tojson
-
         class AssistantTracker(Extension):
             tags = {"generation"}
 
@@ -1989,7 +1986,7 @@ def _generation_support(self, context, caller):
                 return rv
 
         jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, extensions=[AssistantTracker])
-        jinja_env.policies["json.dumps_kwargs"]["ensure_ascii"] = False
+        jinja_env.filters["tojson"] = tojson
         jinja_env.globals["raise_exception"] = raise_exception
         return jinja_env.from_string(chat_template)