When calling the html serializer pass an encoding (#239)

Fixes #238
plone · May 2, 2024 · ebaa95f · ebaa95f
1 parent 32c2994
commit ebaa95f
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 2 deletions.
diff --git a/news/238.bugfix b/news/238.bugfix
@@ -0,0 +1 @@
+Fix an issue with unicode characters happening with lxml 5 [ale-rt]
diff --git a/src/plone/app/theming/tests/test_transform.py b/src/plone/app/theming/tests/test_transform.py
@@ -1,11 +1,13 @@
 from App.config import getConfiguration
 from diazo.compiler import compile_theme
+from html import unescape
 from lxml import etree
 from os import environ
 from plone.app.testing import setRoles
 from plone.app.testing import TEST_USER_ID
 from plone.app.theming.interfaces import IThemeSettings
 from plone.app.theming.testing import THEMING_FUNCTIONAL_TESTING
+from plone.app.theming.testing import THEMING_INTEGRATION_TESTING
 from plone.app.theming.transform import ThemeTransform
 from plone.app.theming.utils import applyTheme
 from plone.app.theming.utils import getTheme
@@ -25,7 +27,30 @@
 import unittest
 
 
-class TestCase(unittest.TestCase):
+class IntegrationTestCase(unittest.TestCase):
+
+    layer = THEMING_INTEGRATION_TESTING
+
+    def test_transform_parseTree_with_unicode(self):
+        request = self.layer["request"]
+        request.response.setHeader("Content-Type", "text/html; charset=utf-8")
+        transform = ThemeTransform(None, request)
+        snippet = "\n".join(
+            (
+                "<!DOCTYPE html>",
+                "<html>",
+                "<body>",
+                "<div>à</div>",
+                "</body>",
+                "</html>",
+            )
+        )
+        parsed = transform.parseTree([snippet.encode()])
+        serialized = unescape(parsed.serialize().decode())
+        self.assertEqual(snippet, serialized)
+
+
+class FunctionalTestCase(unittest.TestCase):
     layer = THEMING_FUNCTIONAL_TESTING
 
     def setUp(self):

diff --git a/src/plone/app/theming/transform.py b/src/plone/app/theming/transform.py
@@ -13,6 +13,7 @@
 from zope.component import adapter
 from zope.interface import implementer
 from zope.interface import Interface
+from ZPublisher.HTTPRequest import default_encoding
 
 import logging
 
@@ -120,7 +121,9 @@ def parseTree(self, result):
             return None
 
         try:
-            return getHTMLSerializer(result, pretty_print=False)
+            return getHTMLSerializer(
+                result, pretty_print=False, encoding=default_encoding
+            )
         except (AttributeError, TypeError, etree.ParseError):
             return None
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fix an issue with unicode characters happening with lxml 5 [ale-rt]