diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index aa118804182536..4520e98c10ae58 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1958,9 +1958,6 @@ def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False) # We also expose some options like custom indents and separators return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys) - jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) - jinja_env.filters["tojson"] = tojson - class AssistantTracker(Extension): tags = {"generation"} @@ -1989,7 +1986,7 @@ def _generation_support(self, context, caller): return rv jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, extensions=[AssistantTracker]) - jinja_env.policies["json.dumps_kwargs"]["ensure_ascii"] = False + jinja_env.filters["tojson"] = tojson jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template)