From 988298d9c9e9c4ce16e387ad6beac528e5c2ad1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A1=B0=EC=A4=80=EB=9E=98?= Date: Wed, 12 Jun 2024 22:59:35 +0900 Subject: [PATCH] Change JSON serialization to custom json.dumps (#31100) * Change JSON serialization to custom json.dumps to prevent escaping of "<", ">", "&", "'" * caller has control over the order, remove sort_key=True * Move tojson into a proper function and expose a couple of other args --------- Co-authored-by: jun.4 Co-authored-by: Matt --- src/transformers/tokenization_utils_base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 14a31560c6f1a8..8f72b303bbbbcc 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -1892,8 +1892,13 @@ def _compile_jinja_template(self, chat_template): def raise_exception(message): raise TemplateError(message) + def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False): + # We override the built-in tojson filter because Jinja's default filter escapes HTML characters + # We also expose some options like custom indents and separators + return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys) + jinja_env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) - jinja_env.policies["json.dumps_kwargs"]["ensure_ascii"] = False + jinja_env.filters["tojson"] = tojson jinja_env.globals["raise_exception"] = raise_exception return jinja_env.from_string(chat_template)