Fix html escaping of source line marks

SourceFileViewHTMLGenerator escaping code had multiple issues: - When marking each line from range_start_pragma_processor as safe, we wrapped not only strings but also whole tuples in Markup(). - For tuple elements we didn't differentiate whether they from pygments (already escaped) or from source file (not yet escaped). Only the later must be explicitly escaped. Some changes are there to make mypy happy. Relates to strictdoc-project#1921.
haxtibal · Jul 28, 2024 · 134b9a1 · 134b9a1
1 parent fd2e364
commit 134b9a1
Show file tree

Hide file tree

Showing 10 changed files with 220 additions and 16 deletions.
diff --git a/strictdoc/export/html/generators/source_file_view_generator.py b/strictdoc/export/html/generators/source_file_view_generator.py
@@ -1,5 +1,5 @@
 # mypy: disable-error-code="no-untyped-call,no-untyped-def,operator"
-from typing import List, Tuple
+from typing import List, Tuple, Union
 
 from markupsafe import Markup, escape
 from pygments import highlight
@@ -28,10 +28,13 @@
 from strictdoc.core.traceability_index import TraceabilityIndex
 from strictdoc.export.html.generators.view_objects.source_file_view_object import (
     SourceFileViewObject,
+    SourceLineEntry,
+    SourceMarkerTuple,
 )
 from strictdoc.export.html.html_templates import HTMLTemplates
 from strictdoc.export.html.renderers.link_renderer import LinkRenderer
 from strictdoc.export.html.renderers.markup_renderer import MarkupRenderer
+from strictdoc.helpers.cast import assert_cast
 
 
 class SourceFileViewHTMLGenerator:
@@ -46,7 +49,7 @@ def export(
         with open(source_file.full_path, encoding="utf-8") as opened_file:
             source_file_lines = opened_file.readlines()
 
-        pygmented_source_file_lines: List[Markup] = []
+        pygmented_source_file_lines: List[SourceLineEntry] = []
         pygments_styles: Markup = Markup("")
 
         if len(source_file_lines) > 0:
@@ -89,7 +92,10 @@ def get_pygmented_source_lines(
         source_file: SourceFile,
         source_file_lines: List[str],
         coverage_info: SourceFileTraceabilityInfo,
-    ) -> Tuple[List[Markup], Markup]:
+    ) -> Tuple[
+        List[SourceLineEntry],
+        Markup,
+    ]:
         assert isinstance(source_file, SourceFile)
         assert isinstance(source_file_lines, list)
         assert isinstance(coverage_info, SourceFileTraceabilityInfo)
@@ -154,7 +160,9 @@ def get_pygmented_source_lines(
         pygmented_source_file_content = pygmented_source_file_content[
             slice_start:slice_end
         ]
-        pygmented_source_file_lines = pygmented_source_file_content.split("\n")
+        pygmented_source_file_lines: List[Union[str, SourceMarkerTuple]] = list(
+            pygmented_source_file_content.split("\n")
+        )
         if hack_first_line:
             pygmented_source_file_lines[0] = "<span></span>"
 
@@ -177,12 +185,14 @@ def get_pygmented_source_lines(
 
         for pragma in coverage_info.pragmas:
             pragma_line = pragma.ng_source_line_begin
+            assert isinstance(pragma_line, int)
+            pygmented_source_file_line = assert_cast(
+                pygmented_source_file_lines[pragma_line - 1], str
+            )
             if isinstance(pragma, ForwardRangeMarker):
+                before_line = pygmented_source_file_line.rstrip("\n") + " "
                 pygmented_source_file_lines[pragma_line - 1] = (
-                    pygmented_source_file_lines[pragma_line - 1].rstrip("\n")
-                    + " ",
-                    "\n",
-                    pragma,
+                    SourceMarkerTuple(Markup(before_line), Markup("\n"), pragma)
                 )
                 continue
 
@@ -202,7 +212,7 @@ def get_pygmented_source_lines(
             assert closing_bracket_index is not None
             after_line = source_line[closing_bracket_index:].rstrip()
 
-            pygmented_source_file_lines[pragma_line - 1] = (
+            pygmented_source_file_lines[pragma_line - 1] = SourceMarkerTuple(
                 escape(before_line),
                 escape(after_line),
                 pragma,
@@ -212,6 +222,13 @@ def get_pygmented_source_lines(
             + html_formatter.get_style_defs(".highlight")
         )
 
-        return list(map(Markup, pygmented_source_file_lines)), Markup(
-            pygments_styles
-        )
+        return [
+            SourceFileViewHTMLGenerator.mark_safe(line)
+            for line in pygmented_source_file_lines
+        ], Markup(pygments_styles)
+
+    @staticmethod
+    def mark_safe(
+        line: Union[str, SourceMarkerTuple],
+    ) -> Union[Markup, SourceMarkerTuple]:
+        return line if isinstance(line, SourceMarkerTuple) else Markup(line)
diff --git a/strictdoc/export/html/generators/view_objects/source_file_view_object.py b/strictdoc/export/html/generators/view_objects/source_file_view_object.py
@@ -1,11 +1,16 @@
 # mypy: disable-error-code="no-any-return,no-untyped-call,no-untyped-def,union-attr"
 from dataclasses import dataclass
 from datetime import datetime
-from typing import List
+from typing import List, NamedTuple, Union
 
 from markupsafe import Markup
 
 from strictdoc import __version__
+from strictdoc.backend.sdoc_source_code.models.range_marker import (
+    ForwardRangeMarker,
+    LineMarker,
+    RangeMarker,
+)
 from strictdoc.core.document_tree_iterator import DocumentTreeIterator
 from strictdoc.core.finders.source_files_finder import SourceFile
 from strictdoc.core.project_config import ProjectConfig
@@ -15,6 +20,18 @@
 from strictdoc.export.html.renderers.markup_renderer import MarkupRenderer
 
 
+class SourceMarkerTuple(NamedTuple):
+    before_line: Markup
+    after_line: Markup
+    pragma: Union[ForwardRangeMarker, LineMarker, RangeMarker]
+
+
+SourceLineEntry = Union[
+    Markup,
+    SourceMarkerTuple,
+]
+
+
 @dataclass
 class SourceFileViewObject:
     def __init__(
@@ -26,15 +43,15 @@ def __init__(
         markup_renderer: MarkupRenderer,
         source_file: SourceFile,
         pygments_styles: Markup,
-        pygmented_source_file_lines: List[Markup],
+        pygmented_source_file_lines: List[SourceLineEntry],
     ):
         self.traceability_index: TraceabilityIndex = traceability_index
         self.project_config: ProjectConfig = project_config
         self.link_renderer: LinkRenderer = link_renderer
         self.markup_renderer: MarkupRenderer = markup_renderer
         self.source_file: SourceFile = source_file
         self.pygments_styles: Markup = pygments_styles
-        self.pygmented_source_file_lines: List[Markup] = (
+        self.pygmented_source_file_lines: List[SourceLineEntry] = (
             pygmented_source_file_lines
         )
 

diff --git a/strictdoc/export/html/templates/screens/source_file_view/main.jinja b/strictdoc/export/html/templates/screens/source_file_view/main.jinja
@@ -6,7 +6,7 @@
     {%- for line in view_object.pygmented_source_file_lines -%}
       <div id="line-{{ loop.index }}" class="source__line-number"><pre>{{ loop.index }}</pre></div>
       <div data-line={{ loop.index }} class="source__line-content">
-        {%- if line.__class__.__name__ == "tuple" -%}
+        {%- if line.__class__.__name__ == "SourceMarkerTuple" -%}
           {%- set replacement_before, replacement_after, pragma = line -%}
           {# Note: Cannot format HTML/Jinja blocks within 'pre' tags! #}
           <pre class="sdoc-comment">{{ replacement_before }}{%- for req in pragma.reqs_objs -%}

diff --git a/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/input.sdoc b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/input.sdoc
@@ -0,0 +1,18 @@
+[DOCUMENT]
+TITLE: <b>"escaping"&nbsp;'document title'</b>
+
+[SECTION]
+TITLE: <b>"escaping"&nbsp;'section title'</b>
+
+[TEXT]
+STATEMENT: >>>
+<b>"escaping"&nbsp;'text statement'</b>
+<<<
+
+[REQUIREMENT]
+UID: REQ-1
+STATEMENT: >>>
+<b>"escaping"&nbsp;'requirement statement'</b>
+<<<
+
+[/SECTION]
diff --git a/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/strictdoc.toml b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/strictdoc.toml
@@ -0,0 +1,8 @@
+[project]
+
+features = [
+  "DEEP_TRACEABILITY_SCREEN",
+  "STANDALONE_DOCUMENT_SCREEN",
+  "TABLE_SCREEN",
+  "TRACEABILITY_SCREEN",
+]
diff --git a/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/test.itest b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/test.itest
@@ -0,0 +1,51 @@
+RUN: %strictdoc export %S --output-dir Output/ | filecheck %s --dump-input=fail --check-prefix CHECK-EXPORT
+CHECK-EXPORT: Published: <b>"escaping"&nbsp;'document title'</b>
+
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-INPUT
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-DEEP-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-DEEP-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-DTR
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TABLE.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TABLE.html | filecheck %s --dump-input=fail --check-prefix CHECK-TABLE
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-TRACE
+
+# Browser title bar: Document title.
+CHECK-ALL: <title>&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;document title&#39;&lt;/b&gt; - {{Document|Deep Traceability|Table|Traceability}}</title>
+
+# Left-hand bar: project tree document entry.
+CHECK-ALL: class="document_title"
+CHECK-ALL-NEXT: title="&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;document title&#39;&lt;/b&gt;"
+CHECK-ALL-NEXT: data-file_name="input.sdoc"
+CHECK-ALL-NEXT: >&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;document title&#39;&lt;/b&gt;</div>
+
+# Right-hand bar: Document TOC.
+CHECK-ALL: <span class="section-number">
+CHECK-ALL-NEXT: {{[0-9]+}}
+CHECK-ALL-NEXT: </span>&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;section title&#39;&lt;/b&gt;
+
+# Header: Document path in tree.
+CHECK-ALL: class="header__document_title"
+CHECK-ALL-NEXT: title="&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;document title&#39;&lt;/b&gt;"
+CHECK-ALL-NEXT: >
+CHECK-ALL-NEXT: &lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;document title&#39;&lt;/b&gt;
+
+# Main document: Title.
+CHECK-INPUT: <h1 data-testid="document-title">&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;document title&#39;&lt;/b&gt;</h1>
+CHECK-TABLE: <h1 data-testid="document-title">&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;document title&#39;&lt;/b&gt;</h1>
+
+# Main document: Section.
+CHECK-INPUT: <sdoc-autogen>1.&nbsp;&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;section title&#39;&lt;/b&gt;</sdoc-autogen>
+CHECK-DTR: <sdoc-autogen>1.&nbsp;&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;section title&#39;&lt;/b&gt;</sdoc-autogen>
+CHECK-TABLE: <div class="requirement__title">
+CHECK-TABLE-NEXT: &lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;section title&#39;&lt;/b&gt;
+CHECK-TABLE-NEXT: </div>
+CHECK-TRACE: <sdoc-autogen>1.&nbsp;&lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;section title&#39;&lt;/b&gt;</sdoc-autogen>
+
+# Main document: Text statement.
+CHECK-INPUT: <p>&lt;b&gt;&quot;escaping&quot;&amp;nbsp;'text statement'&lt;/b&gt;</p>
+CHECK-TABLE: <p>&lt;b&gt;&quot;escaping&quot;&amp;nbsp;'text statement'&lt;/b&gt;</p>
+CHECK-TRACE: <p>&lt;b&gt;&quot;escaping&quot;&amp;nbsp;'text statement'&lt;/b&gt;</p>
+
+# Main document: Requirement statement.
+CHECK-ALL: <p>&lt;b&gt;&quot;escaping&quot;&amp;nbsp;'requirement statement'&lt;/b&gt;</p>
diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/file.py b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/file.py
@@ -0,0 +1,15 @@
+# <b>"escaping"&nbsp;'line mark'</b> @sdoc(REQ-1)
+def print_test():
+    test1 = """
+    <b>"escaping"&nbsp;'normal src line'</b>
+    """
+    test2 = """<b>"escaping"&nbsp;'forward range mark before'</b>
+    <b>"escaping"&nbsp;'forward range mark after'</b>
+    """
+    print(f"{test1} {test2}")  # noqa: T201#
+
+
+# <b>"escaping"&nbsp;'range mark before'</b> @sdoc[REQ-1]
+def hello_world():
+    print("hello world")  # noqa: T201
+# <b>"escaping"&nbsp;'range mark after'</b> @sdoc[/REQ-1]
diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/input.sdoc b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/input.sdoc
@@ -0,0 +1,14 @@
+[DOCUMENT]
+TITLE: HTML escaping of source file content
+
+[REQUIREMENT]
+UID: REQ-1
+STATEMENT: >>>
+Source files are external input, their content must be HTML escaped.
+<<<
+RELATIONS:
+- TYPE: File
+  VALUE: file.py
+- TYPE: File
+  VALUE: file.py
+  LINE_RANGE: 6, 7
diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/strictdoc.toml b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/strictdoc.toml
@@ -0,0 +1,5 @@
+[project]
+
+features = [
+  "REQUIREMENT_TO_SOURCE_TRACEABILITY",
+]
diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/test.itest b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/test.itest
@@ -0,0 +1,59 @@
+RUN: %strictdoc export %S --output-dir Output/ | filecheck %s --dump-input=fail --check-prefix CHECK-EXPORT
+CHECK-EXPORT: Published: HTML escaping of source file content
+
+RUN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SRC
+
+# Line marker.
+CHECK-SRC: <pre class="sdoc-comment"># &lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;line mark&#39;&lt;/b&gt; @sdoc(<a
+CHECK-SRC-NEXT: class="pointer"
+CHECK-SRC-NEXT: data-reqid="REQ-1"
+CHECK-SRC-NEXT: data-begin="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-end="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-traceability-file-type="this_file"
+CHECK-SRC-NEXT: href="../_source_files/file.py.html#REQ-1#{{[0-9]+}}#{{[0-9]+}}"
+CHECK-SRC-NEXT: >REQ-1</a>)</pre></div><div id="line-{{[0-9]+}}" class="source__line-number"><pre>2</pre>
+
+# Normal source code line.
+CHECK-SRC: <pre class="highlight"><span class="s2">{{\s+}}&lt;b&gt;&quot;escaping&quot;&amp;nbsp;&#39;normal src line&#39;&lt;/b&gt;</span></pre>
+
+# Forward range marker before.
+CHECK-SRC: <pre class="sdoc-comment">{{\s+}}<span class="n">test2</span>{{\s+}}<span class="o">=</span> <span class="s2">&quot;&quot;&quot;&lt;b&gt;&quot;escaping&quot;&amp;nbsp;&#39;forward range mark before&#39;&lt;/b&gt;</span>{{\s+}}<a
+CHECK-SRC-NEXT: class="pointer"
+CHECK-SRC-NEXT: data-reqid="REQ-1"
+CHECK-SRC-NEXT: data-begin="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-end="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-traceability-file-type="this_file"
+CHECK-SRC-NEXT: href="../_source_files/file.py.html#REQ-1#{{[0-9]+}}#{{[0-9]+}}"
+CHECK-SRC-NEXT: >REQ-1</a>
+CHECK-SRC-NEXT: </pre>
+
+# Forward range marker after.
+CHECK-SRC: <pre class="sdoc-comment"><span class="s2">{{\s+}}&lt;b&gt;&quot;escaping&quot;&amp;nbsp;&#39;forward range mark after&#39;&lt;/b&gt;</span>{{\s+}}<a
+CHECK-SRC-NEXT: class="pointer"
+CHECK-SRC-NEXT: data-reqid="REQ-1"
+CHECK-SRC-NEXT: data-begin="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-end="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-traceability-file-type="this_file"
+CHECK-SRC-NEXT: href="../_source_files/file.py.html#REQ-1#{{[0-9]+}}#{{[0-9]+}}"
+CHECK-SRC-NEXT: >/REQ-1</a>
+CHECK-SRC-NEXT: </pre>
+
+# Range marker before.
+CHECK-SRC: <pre class="sdoc-comment"># &lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;range mark before&#39;&lt;/b&gt; @sdoc[<a
+CHECK-SRC-NEXT: class="pointer"
+CHECK-SRC-NEXT: data-reqid="REQ-1"
+CHECK-SRC-NEXT: data-begin="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-end="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-traceability-file-type="this_file"
+CHECK-SRC-NEXT: href="../_source_files/file.py.html#REQ-1#{{[0-9]+}}#{{[0-9]+}}"
+CHECK-SRC-NEXT: >REQ-1</a>]</pre>
+
+# Range marker after.
+CHECK-SRC: <pre class="sdoc-comment"># &lt;b&gt;&#34;escaping&#34;&amp;nbsp;&#39;range mark after&#39;&lt;/b&gt; @sdoc[<a
+CHECK-SRC-NEXT: class="pointer"
+CHECK-SRC-NEXT: data-reqid="REQ-1"
+CHECK-SRC-NEXT: data-begin="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-end="{{[0-9]+}}"
+CHECK-SRC-NEXT: data-traceability-file-type="this_file"
+CHECK-SRC-NEXT: href="../_source_files/file.py.html#REQ-1#{{[0-9]+}}#{{[0-9]+}}"
+CHECK-SRC-NEXT: >/REQ-1</a>]</pre>