From aecfdd2629873d9810fa6675f43817f6d78db0e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 26 May 2024 10:41:01 +0200 Subject: [PATCH 1/2] Render chat template tojson filter as unicode --- src/transformers/tokenization_utils_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index a8d35003287e39..f67bab852f895d 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1853,6 +1853,7 @@ def raise_exception(message): raise TemplateError(message) jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) + jinja_env.policies['json.dumps_kwargs']['ensure_ascii'] = False jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template) From c9c1bce7ad3838dd058351e4b23a90de66782d2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Sun, 26 May 2024 11:28:21 +0200 Subject: [PATCH 2/2] ruff-- --- src/transformers/tokenization_utils_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index f67bab852f895d..116fbfdf7bbbf0 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1853,7 +1853,7 @@ def raise_exception(message): raise TemplateError(message) jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) - jinja_env.policies['json.dumps_kwargs']['ensure_ascii'] = False + jinja_env.policies["json.dumps_kwargs"]["ensure_ascii"] = False jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template)