Merge pull request #1921 from haxtibal/tdmg/escape_singleline_html

Use Jinja2 autoescaping

docs/strictdoc_25_design.sdoc

@@ -136,3 +136,39 @@ One important implementation detail of Arpeggio that influences StrictDoc user e
 [/FREETEXT]
 
 [/SECTION]
+
+[SECTION]
+TITLE: HTML Escaping
+
+[TEXT]
+STATEMENT: >>>
+StrictDoc uses Jinja2 autoescaping_ for HTML output. `Template.render`_ calls
+will escape any Python object unless it's explicitly marked as safe.
+
+Good to know for a start:
+
+- If a Python object intentionally contains HTML it must be marked as safe
+  to bypass autoescaping. Templates can do this by piping to safe_, or Python code
+  can do it by wrapping an object into `markupsafe.Markup`_.
+- Passing text to the `Markup() <markupsafe.Markup_>`_ constructor marks that text
+  as safe, but *does not escape* it.
+- Text can be explicitly escaped with `markupsafe.escape`_. It's similar to
+  `html.escape`_, but the result is immediately marked safe.
+- `markupsafe.Markup`_ is responsible for some "magic". It's a :code:`str` subclass
+  with the same methods, but escaping arguments. For example,
+  :code:`"> " + Markup("<div>safe</div>")` will turn into :code:`"&gt; <div>safe</div>"`,
+  thanks to :code:`__radd__` in this specific case. To prevent escaping,
+  you would use :code:`Markup("> ") + Markup("<div>safe</div>")`. Basically the
+  same magic happens in templates when using safe_.
+- See also `Working with Automatic Escaping`_.
+
+.. _autoescaping: https://jinja.palletsprojects.com/en/latest/api/#autoescaping
+.. _Working with Automatic Escaping: https://jinja.palletsprojects.com/en/latest/templates/#working-with-automatic-escaping
+.. _markupsafe.Markup: https://markupsafe.palletsprojects.com/en/latest/escaping/#markupsafe.Markup
+.. _markupsafe.escape: https://markupsafe.palletsprojects.com/en/latest/escaping/#markupsafe.escape
+.. _safe: https://jinja.palletsprojects.com/en/latest/templates/#jinja-filters.safe
+.. _Template.render: https://jinja.palletsprojects.com/en/latest/api/#jinja2.Template.render
+.. _html.escape: https://docs.python.org/3/library/html.html#html.escape
+<<<
+
+[/SECTION]

strictdoc/backend/sdoc/models/free_text.py

@@ -1,4 +1,3 @@
-import html
 from typing import Any, List, Optional
 
 from strictdoc.backend.sdoc.models.anchor import Anchor
@@ -55,9 +54,6 @@ def get_parts_as_text(self) -> str:
                 raise NotImplementedError(part)
         return text
 
-    def get_parts_as_text_escaped(self) -> str:
-        return html.escape(self.get_parts_as_text())
-
 
 class FreeTextContainer(FreeText):
     def __init__(self, parts: List[Any]) -> None:

strictdoc/backend/sdoc/models/node.py

@@ -1,5 +1,4 @@
 # mypy: disable-error-code="union-attr"
-import html
 from collections import OrderedDict
 from typing import Any, Generator, List, Optional, Tuple, Union
 
@@ -92,9 +91,6 @@ def get_text_value(self) -> str:
                 raise NotImplementedError(part)
         return text
 
-    def get_text_value_escaped(self) -> str:
-        return html.escape(self.get_text_value())
-
 
 @auto_described
 class SDocNode(SDocObject):
@@ -456,13 +452,6 @@ def enumerate_all_fields(
             meta_field_value = field.get_text_value()
             yield field, field.field_name, meta_field_value
 
-    def enumerate_all_fields_escaped(
-        self,
-    ) -> Generator[Tuple[SDocNodeField, str, str], None, None]:
-        for field in self.enumerate_fields():
-            meta_field_value = field.get_text_value_escaped()
-            yield field, field.field_name, meta_field_value
-
     def enumerate_meta_fields(
         self, skip_single_lines: bool = False, skip_multi_lines: bool = False
     ) -> Generator[Tuple[str, SDocNodeField], None, None]:

strictdoc/core/transforms/create_requirement.py

@@ -108,14 +108,14 @@ def perform(self):
                 ) = RstToHtmlFragmentWriter(
                     path_to_output_dir=self.project_config.export_output_dir,
                     context_document=document,
-                ).write_with_validation(field_.field_unescaped_value)
+                ).write_with_validation(field_.field_value)
                 if parsed_html is None:
                     errors[field_.field_name].append(rst_error)
                 else:
                     try:
                         free_text_container: Optional[FreeTextContainer] = (
-                            SDFreeTextReader.read(field_.field_unescaped_value)
-                            if len(field_.field_unescaped_value) > 0
+                            SDFreeTextReader.read(field_.field_value)
+                            if len(field_.field_value) > 0
                             else None
                         )
                         map_form_to_requirement_fields[field_] = (
@@ -208,7 +208,7 @@ def perform(self):
                     requirement.set_field_value(
                         field_name=form_field_name,
                         form_field_index=form_field_index,
-                        value=form_field.field_unescaped_value,
+                        value=form_field.field_value,
                     )
                     continue
 

strictdoc/core/transforms/update_requirement.py

@@ -114,14 +114,14 @@ def perform(self):
                 ) = RstToHtmlFragmentWriter(
                     path_to_output_dir=self.project_config.export_output_dir,
                     context_document=self.context_document,
-                ).write_with_validation(field_.field_unescaped_value)
+                ).write_with_validation(field_.field_value)
                 if parsed_html is None:
                     form_object.add_error(field_.field_name, rst_error)
                 else:
                     try:
                         free_text_container: Optional[FreeTextContainer] = (
-                            SDFreeTextReader.read(field_.field_unescaped_value)
-                            if len(field_.field_unescaped_value) > 0
+                            SDFreeTextReader.read(field_.field_value)
+                            if len(field_.field_value) > 0
                             else None
                         )
                         map_form_to_requirement_fields[field_] = (
@@ -455,7 +455,7 @@ def populate_node_fields_from_form_object(
                     node.set_field_value(
                         field_name=form_field_name,
                         form_field_index=form_field_index,
-                        value=form_field.field_unescaped_value,
+                        value=form_field.field_value,
                     )
                     continue
 

strictdoc/export/html/form_objects/form_object.py

@@ -2,25 +2,25 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List
 
-from jinja2 import Environment, Template
+from strictdoc.export.html.html_templates import JinjaEnvironment
 
 
 @dataclass
 class RowWithReservedFieldFormObject:
     field: Any
     errors: Dict[str, List]
-    jinja_environment: Environment
+    jinja_environment: JinjaEnvironment
 
     def __post_init__(self):
         assert isinstance(
-            self.jinja_environment, Environment
+            self.jinja_environment, JinjaEnvironment
         ), self.jinja_environment
 
     def render(self):
-        template: Template = self.jinja_environment.get_template(
-            "components/grammar_form_element/row_with_reserved_field/index.jinja"
+        rendered_template = self.jinja_environment.render_template_as_markup(
+            "components/grammar_form_element/row_with_reserved_field/index.jinja",
+            form_object=self,
         )
-        rendered_template = template.render(form_object=self)
         return rendered_template
 
     def get_errors(self, field_name) -> List:
@@ -31,19 +31,19 @@ def get_errors(self, field_name) -> List:
 class RowWithCustomFieldFormObject:
     field: Any
     errors: Dict[str, List]
-    jinja_environment: Environment
+    jinja_environment: JinjaEnvironment
 
     def __post_init__(self):
         assert self.field is not None
         assert isinstance(
-            self.jinja_environment, Environment
+            self.jinja_environment, JinjaEnvironment
         ), self.jinja_environment
 
     def render(self):
-        template: Template = self.jinja_environment.get_template(
-            "components/grammar_form_element/row_with_custom_field/index.jinja"
+        rendered_template = self.jinja_environment.render_template_as_markup(
+            "components/grammar_form_element/row_with_custom_field/index.jinja",
+            form_object=self,
         )
-        rendered_template = template.render(form_object=self)
         return rendered_template
 
     def get_errors(self, field_name) -> List:
@@ -54,19 +54,19 @@ def get_errors(self, field_name) -> List:
 class RowWithRelationFormObject:
     relation: Any
     errors: Dict[str, List]
-    jinja_environment: Environment
+    jinja_environment: JinjaEnvironment
 
     def __post_init__(self):
         assert self.relation is not None
         assert isinstance(
-            self.jinja_environment, Environment
+            self.jinja_environment, JinjaEnvironment
         ), self.jinja_environment
 
     def render(self):
-        template: Template = self.jinja_environment.get_template(
-            "components/grammar_form_element/row_with_relation/index.jinja"
+        rendered_template = self.jinja_environment.render_template_as_markup(
+            "components/grammar_form_element/row_with_relation/index.jinja",
+            form_object=self,
         )
-        rendered_template = template.render(form_object=self)
         return rendered_template
 
     def get_errors(self, field_name) -> List:

strictdoc/export/html/form_objects/grammar_element_form_object.py

@@ -1,7 +1,7 @@
 # mypy: disable-error-code="arg-type,no-any-return,no-untyped-call,no-untyped-def,union-attr,type-arg"
 from typing import Dict, List, Optional, Set, Tuple, Union
 
-from jinja2 import Environment, Template
+from jinja2 import Template
 from starlette.datastructures import FormData
 
 from strictdoc.backend.sdoc.models.document import SDocDocument
@@ -23,6 +23,7 @@
     RowWithRelationFormObject,
     RowWithReservedFieldFormObject,
 )
+from strictdoc.export.html.html_templates import JinjaEnvironment
 from strictdoc.helpers.auto_described import auto_described
 from strictdoc.helpers.cast import assert_cast
 from strictdoc.helpers.form_data import parse_form_data
@@ -108,7 +109,7 @@ def __init__(
         fields: List[GrammarFormField],
         relations: List[GrammarFormRelation],
         project_config: ProjectConfig,
-        jinja_environment: Environment,
+        jinja_environment: JinjaEnvironment,
     ):
         assert isinstance(document_mid, str), document_mid
         super().__init__()
@@ -118,15 +119,15 @@ def __init__(
         self.fields: List[GrammarFormField] = fields
         self.relations: List[GrammarFormRelation] = relations
         self.project_config: ProjectConfig = project_config
-        self.jinja_environment: Environment = jinja_environment
+        self.jinja_environment: JinjaEnvironment = jinja_environment
 
     @staticmethod
     def create_from_request(
         *,
         document: SDocDocument,
         request_form_data: FormData,
         project_config: ProjectConfig,
-        jinja_environment: Environment,
+        jinja_environment: JinjaEnvironment,
     ) -> "GrammarElementFormObject":
         form_object_fields: List[GrammarFormField] = []
         form_object_relations: List[GrammarFormRelation] = []
@@ -193,7 +194,7 @@ def create_from_document(
         document: SDocDocument,
         element_mid: str,
         project_config: ProjectConfig,
-        jinja_environment: Environment,
+        jinja_environment: JinjaEnvironment,
     ) -> "GrammarElementFormObject":
         assert isinstance(document, SDocDocument)
         assert isinstance(document.grammar, DocumentGrammar)
@@ -371,10 +372,9 @@ def render(self):
         )
 
     def render_after_validation(self):
-        template: Template = self.jinja_environment.get_template(
-            "components/grammar_form_element/index.jinja"
+        rendered_template = self.jinja_environment.render_template_as_markup(
+            "components/grammar_form_element/index.jinja", form_object=self
         )
-        rendered_template = template.render(form_object=self)
         return render_turbo_stream(
             content=rendered_template, action="update", target="modal"
         )

strictdoc/export/html/form_objects/grammar_form_object.py

@@ -1,7 +1,6 @@
 # mypy: disable-error-code="arg-type,no-any-return,no-untyped-call,no-untyped-def,type-arg"
 from typing import Dict, List, Optional, Set
 
-from jinja2 import Environment, Template
 from starlette.datastructures import FormData
 
 from strictdoc.backend.sdoc.models.document import SDocDocument
@@ -13,6 +12,7 @@
 from strictdoc.export.html.form_objects.rows.row_with_grammar_element_form_object import (
     RowWithGrammarElementFormObject,
 )
+from strictdoc.export.html.html_templates import JinjaEnvironment
 from strictdoc.helpers.auto_described import auto_described
 from strictdoc.helpers.cast import assert_cast
 from strictdoc.helpers.form_data import parse_form_data
@@ -68,15 +68,15 @@ def __init__(
         document_mid: str,
         fields: List[GrammarElementFormField],
         project_config: ProjectConfig,
-        jinja_environment: Environment,
+        jinja_environment: JinjaEnvironment,
         imported_grammar_file: Optional[str],
     ):
         assert isinstance(document_mid, str), document_mid
         super().__init__()
         self.document_mid = document_mid
         self.fields: List[GrammarElementFormField] = fields
         self.project_config: ProjectConfig = project_config
-        self.jinja_environment: Environment = jinja_environment
+        self.jinja_environment: JinjaEnvironment = jinja_environment
         self.imported_grammar_file: Optional[str] = imported_grammar_file
 
     @staticmethod
@@ -85,7 +85,7 @@ def create_from_request(
         document_mid: str,
         request_form_data: FormData,
         project_config: ProjectConfig,
-        jinja_environment: Environment,
+        jinja_environment: JinjaEnvironment,
     ) -> "GrammarFormObject":
         form_object_fields: List[GrammarElementFormField] = []
         request_form_data_as_list = [
@@ -125,7 +125,7 @@ def create_from_document(
         *,
         document: SDocDocument,
         project_config: ProjectConfig,
-        jinja_environment: Environment,
+        jinja_environment: JinjaEnvironment,
     ) -> "GrammarFormObject":
         assert isinstance(document, SDocDocument)
         assert isinstance(document.grammar, DocumentGrammar)
@@ -180,10 +180,9 @@ def validate(self) -> bool:
         return len(self.errors) == 0
 
     def render(self):
-        template: Template = self.jinja_environment.get_template(
-            "components/grammar_form/index.jinja"
+        rendered_template = self.jinja_environment.render_template_as_markup(
+            "components/grammar_form/index.jinja", form_object=self
         )
-        rendered_template = template.render(form_object=self)
         return render_turbo_stream(
             content=rendered_template, action="update", target="modal"
         )