From 134b9a1d3e4d283bb15c685c11b4c5faf1c6667e Mon Sep 17 00:00:00 2001 From: Tobias Deiminger Date: Fri, 26 Jul 2024 13:29:20 +0200 Subject: [PATCH] Fix html escaping of source line marks SourceFileViewHTMLGenerator escaping code had multiple issues: - When marking each line from range_start_pragma_processor as safe, we wrapped not only strings but also whole tuples in Markup(). - For tuple elements we didn't differentiate whether they from pygments (already escaped) or from source file (not yet escaped). Only the later must be explicitly escaped. Some changes are there to make mypy happy. Relates to #1921. --- .../generators/source_file_view_generator.py | 41 +++++++++---- .../view_objects/source_file_view_object.py | 23 +++++++- .../screens/source_file_view/main.jinja | 2 +- .../01_escape_input_from_sdoc/input.sdoc | 18 ++++++ .../01_escape_input_from_sdoc/strictdoc.toml | 8 +++ .../01_escape_input_from_sdoc/test.itest | 51 ++++++++++++++++ .../02_escape_input_from_src_file/file.py | 15 +++++ .../02_escape_input_from_src_file/input.sdoc | 14 +++++ .../strictdoc.toml | 5 ++ .../02_escape_input_from_src_file/test.itest | 59 +++++++++++++++++++ 10 files changed, 220 insertions(+), 16 deletions(-) create mode 100644 tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/input.sdoc create mode 100644 tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/strictdoc.toml create mode 100644 tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/test.itest create mode 100644 tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/file.py create mode 100644 tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/input.sdoc create mode 100644 tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/strictdoc.toml create mode 100644 tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/test.itest diff --git a/strictdoc/export/html/generators/source_file_view_generator.py b/strictdoc/export/html/generators/source_file_view_generator.py index de36ccde1..ac485ce19 100644 --- a/strictdoc/export/html/generators/source_file_view_generator.py +++ b/strictdoc/export/html/generators/source_file_view_generator.py @@ -1,5 +1,5 @@ # mypy: disable-error-code="no-untyped-call,no-untyped-def,operator" -from typing import List, Tuple +from typing import List, Tuple, Union from markupsafe import Markup, escape from pygments import highlight @@ -28,10 +28,13 @@ from strictdoc.core.traceability_index import TraceabilityIndex from strictdoc.export.html.generators.view_objects.source_file_view_object import ( SourceFileViewObject, + SourceLineEntry, + SourceMarkerTuple, ) from strictdoc.export.html.html_templates import HTMLTemplates from strictdoc.export.html.renderers.link_renderer import LinkRenderer from strictdoc.export.html.renderers.markup_renderer import MarkupRenderer +from strictdoc.helpers.cast import assert_cast class SourceFileViewHTMLGenerator: @@ -46,7 +49,7 @@ def export( with open(source_file.full_path, encoding="utf-8") as opened_file: source_file_lines = opened_file.readlines() - pygmented_source_file_lines: List[Markup] = [] + pygmented_source_file_lines: List[SourceLineEntry] = [] pygments_styles: Markup = Markup("") if len(source_file_lines) > 0: @@ -89,7 +92,10 @@ def get_pygmented_source_lines( source_file: SourceFile, source_file_lines: List[str], coverage_info: SourceFileTraceabilityInfo, - ) -> Tuple[List[Markup], Markup]: + ) -> Tuple[ + List[SourceLineEntry], + Markup, + ]: assert isinstance(source_file, SourceFile) assert isinstance(source_file_lines, list) assert isinstance(coverage_info, SourceFileTraceabilityInfo) @@ -154,7 +160,9 @@ def get_pygmented_source_lines( pygmented_source_file_content = pygmented_source_file_content[ slice_start:slice_end ] - pygmented_source_file_lines = pygmented_source_file_content.split("\n") + pygmented_source_file_lines: List[Union[str, SourceMarkerTuple]] = list( + pygmented_source_file_content.split("\n") + ) if hack_first_line: pygmented_source_file_lines[0] = "" @@ -177,12 +185,14 @@ def get_pygmented_source_lines( for pragma in coverage_info.pragmas: pragma_line = pragma.ng_source_line_begin + assert isinstance(pragma_line, int) + pygmented_source_file_line = assert_cast( + pygmented_source_file_lines[pragma_line - 1], str + ) if isinstance(pragma, ForwardRangeMarker): + before_line = pygmented_source_file_line.rstrip("\n") + " " pygmented_source_file_lines[pragma_line - 1] = ( - pygmented_source_file_lines[pragma_line - 1].rstrip("\n") - + " ", - "\n", - pragma, + SourceMarkerTuple(Markup(before_line), Markup("\n"), pragma) ) continue @@ -202,7 +212,7 @@ def get_pygmented_source_lines( assert closing_bracket_index is not None after_line = source_line[closing_bracket_index:].rstrip() - pygmented_source_file_lines[pragma_line - 1] = ( + pygmented_source_file_lines[pragma_line - 1] = SourceMarkerTuple( escape(before_line), escape(after_line), pragma, @@ -212,6 +222,13 @@ def get_pygmented_source_lines( + html_formatter.get_style_defs(".highlight") ) - return list(map(Markup, pygmented_source_file_lines)), Markup( - pygments_styles - ) + return [ + SourceFileViewHTMLGenerator.mark_safe(line) + for line in pygmented_source_file_lines + ], Markup(pygments_styles) + + @staticmethod + def mark_safe( + line: Union[str, SourceMarkerTuple], + ) -> Union[Markup, SourceMarkerTuple]: + return line if isinstance(line, SourceMarkerTuple) else Markup(line) diff --git a/strictdoc/export/html/generators/view_objects/source_file_view_object.py b/strictdoc/export/html/generators/view_objects/source_file_view_object.py index 9ff0dbbb8..d5dea0060 100644 --- a/strictdoc/export/html/generators/view_objects/source_file_view_object.py +++ b/strictdoc/export/html/generators/view_objects/source_file_view_object.py @@ -1,11 +1,16 @@ # mypy: disable-error-code="no-any-return,no-untyped-call,no-untyped-def,union-attr" from dataclasses import dataclass from datetime import datetime -from typing import List +from typing import List, NamedTuple, Union from markupsafe import Markup from strictdoc import __version__ +from strictdoc.backend.sdoc_source_code.models.range_marker import ( + ForwardRangeMarker, + LineMarker, + RangeMarker, +) from strictdoc.core.document_tree_iterator import DocumentTreeIterator from strictdoc.core.finders.source_files_finder import SourceFile from strictdoc.core.project_config import ProjectConfig @@ -15,6 +20,18 @@ from strictdoc.export.html.renderers.markup_renderer import MarkupRenderer +class SourceMarkerTuple(NamedTuple): + before_line: Markup + after_line: Markup + pragma: Union[ForwardRangeMarker, LineMarker, RangeMarker] + + +SourceLineEntry = Union[ + Markup, + SourceMarkerTuple, +] + + @dataclass class SourceFileViewObject: def __init__( @@ -26,7 +43,7 @@ def __init__( markup_renderer: MarkupRenderer, source_file: SourceFile, pygments_styles: Markup, - pygmented_source_file_lines: List[Markup], + pygmented_source_file_lines: List[SourceLineEntry], ): self.traceability_index: TraceabilityIndex = traceability_index self.project_config: ProjectConfig = project_config @@ -34,7 +51,7 @@ def __init__( self.markup_renderer: MarkupRenderer = markup_renderer self.source_file: SourceFile = source_file self.pygments_styles: Markup = pygments_styles - self.pygmented_source_file_lines: List[Markup] = ( + self.pygmented_source_file_lines: List[SourceLineEntry] = ( pygmented_source_file_lines ) diff --git a/strictdoc/export/html/templates/screens/source_file_view/main.jinja b/strictdoc/export/html/templates/screens/source_file_view/main.jinja index 1e627a751..52a5696ae 100644 --- a/strictdoc/export/html/templates/screens/source_file_view/main.jinja +++ b/strictdoc/export/html/templates/screens/source_file_view/main.jinja @@ -6,7 +6,7 @@ {%- for line in view_object.pygmented_source_file_lines -%} 
{{ loop.index }}
- {%- if line.__class__.__name__ == "tuple" -%} + {%- if line.__class__.__name__ == "SourceMarkerTuple" -%} {%- set replacement_before, replacement_after, pragma = line -%} {# Note: Cannot format HTML/Jinja blocks within 'pre' tags! #} 
{{ replacement_before }}{%- for req in pragma.reqs_objs -%}
diff --git a/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/input.sdoc b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/input.sdoc
new file mode 100644
index 000000000..ccd9a4f2d
--- /dev/null
+++ b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/input.sdoc
@@ -0,0 +1,18 @@
+[DOCUMENT]
+TITLE: "escaping" 'document title'
+
+[SECTION]
+TITLE: "escaping" 'section title'
+
+[TEXT]
+STATEMENT: >>>
+"escaping" 'text statement'
+<<<
+
+[REQUIREMENT]
+UID: REQ-1
+STATEMENT: >>>
+"escaping" 'requirement statement'
+<<<
+
+[/SECTION]
diff --git a/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/strictdoc.toml b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/strictdoc.toml
new file mode 100644
index 000000000..24a5dae1f
--- /dev/null
+++ b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/strictdoc.toml
@@ -0,0 +1,8 @@
+[project]
+
+features = [
+  "DEEP_TRACEABILITY_SCREEN",
+  "STANDALONE_DOCUMENT_SCREEN",
+  "TABLE_SCREEN",
+  "TRACEABILITY_SCREEN",
+]
diff --git a/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/test.itest b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/test.itest
new file mode 100644
index 000000000..eb1d42d4c
--- /dev/null
+++ b/tests/integration/commands/export/html/escaping/01_escape_input_from_sdoc/test.itest
@@ -0,0 +1,51 @@
+RUN: %strictdoc export %S --output-dir Output/ | filecheck %s --dump-input=fail --check-prefix CHECK-EXPORT
+CHECK-EXPORT: Published: "escaping" 'document title'
+
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input.html | filecheck %s --dump-input=fail --check-prefix CHECK-INPUT
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-DEEP-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-DEEP-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-DTR
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TABLE.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TABLE.html | filecheck %s --dump-input=fail --check-prefix CHECK-TABLE
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-ALL
+RUN: %cat %S/Output/html/%THIS_TEST_FOLDER/input-TRACE.html | filecheck %s --dump-input=fail --check-prefix CHECK-TRACE
+
+# Browser title bar: Document title.
+CHECK-ALL: <b>"escaping"&nbsp;'document title'</b> - {{Document|Deep Traceability|Table|Traceability}}
+
+# Left-hand bar: project tree document entry.
+CHECK-ALL: class="document_title"
+CHECK-ALL-NEXT: title="<b>"escaping"&nbsp;'document title'</b>"
+CHECK-ALL-NEXT: data-file_name="input.sdoc"
+CHECK-ALL-NEXT: ><b>"escaping"&nbsp;'document title'</b>
+ +# Right-hand bar: Document TOC. +CHECK-ALL: +CHECK-ALL-NEXT: {{[0-9]+}} +CHECK-ALL-NEXT: <b>"escaping"&nbsp;'section title'</b> + +# Header: Document path in tree. +CHECK-ALL: class="header__document_title" +CHECK-ALL-NEXT: title="<b>"escaping"&nbsp;'document title'</b>" +CHECK-ALL-NEXT: > +CHECK-ALL-NEXT: <b>"escaping"&nbsp;'document title'</b> + +# Main document: Title. +CHECK-INPUT:

<b>"escaping"&nbsp;'document title'</b>

+CHECK-TABLE:

<b>"escaping"&nbsp;'document title'</b>

+ +# Main document: Section. +CHECK-INPUT: 1. <b>"escaping"&nbsp;'section title'</b> +CHECK-DTR: 1. <b>"escaping"&nbsp;'section title'</b> +CHECK-TABLE:
+CHECK-TABLE-NEXT: <b>"escaping"&nbsp;'section title'</b> +CHECK-TABLE-NEXT:
+CHECK-TRACE: 1. <b>"escaping"&nbsp;'section title'</b> + +# Main document: Text statement. +CHECK-INPUT:

<b>"escaping"&nbsp;'text statement'</b>

+CHECK-TABLE:

<b>"escaping"&nbsp;'text statement'</b>

+CHECK-TRACE:

<b>"escaping"&nbsp;'text statement'</b>

+ +# Main document: Requirement statement. +CHECK-ALL:

<b>"escaping"&nbsp;'requirement statement'</b>

diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/file.py b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/file.py new file mode 100644 index 000000000..224963ac4 --- /dev/null +++ b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/file.py @@ -0,0 +1,15 @@ +# "escaping" 'line mark' @sdoc(REQ-1) +def print_test(): + test1 = """ + "escaping" 'normal src line' + """ + test2 = """"escaping" 'forward range mark before' + "escaping" 'forward range mark after' + """ + print(f"{test1} {test2}") # noqa: T201# + + +# "escaping" 'range mark before' @sdoc[REQ-1] +def hello_world(): + print("hello world") # noqa: T201 +# "escaping" 'range mark after' @sdoc[/REQ-1] diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/input.sdoc b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/input.sdoc new file mode 100644 index 000000000..67042fa1a --- /dev/null +++ b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/input.sdoc @@ -0,0 +1,14 @@ +[DOCUMENT] +TITLE: HTML escaping of source file content + +[REQUIREMENT] +UID: REQ-1 +STATEMENT: >>> +Source files are external input, their content must be HTML escaped. +<<< +RELATIONS: +- TYPE: File + VALUE: file.py +- TYPE: File + VALUE: file.py + LINE_RANGE: 6, 7 diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/strictdoc.toml b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/strictdoc.toml new file mode 100644 index 000000000..943e69f5a --- /dev/null +++ b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/strictdoc.toml @@ -0,0 +1,5 @@ +[project] + +features = [ + "REQUIREMENT_TO_SOURCE_TRACEABILITY", +] diff --git a/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/test.itest b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/test.itest new file mode 100644 index 000000000..4f96dc0f6 --- /dev/null +++ b/tests/integration/commands/export/html/escaping/02_escape_input_from_src_file/test.itest @@ -0,0 +1,59 @@ +RUN: %strictdoc export %S --output-dir Output/ | filecheck %s --dump-input=fail --check-prefix CHECK-EXPORT +CHECK-EXPORT: Published: HTML escaping of source file content + +RUN: %cat %S/Output/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SRC + +# Line marker. +CHECK-SRC: 
# <b>"escaping"&nbsp;'line mark'</b> @sdoc(REQ-1)
2
+ +# Normal source code line. +CHECK-SRC: 
{{\s+}}<b>"escaping"&nbsp;'normal src line'</b>
+ +# Forward range marker before. +CHECK-SRC: 
{{\s+}}test2{{\s+}}= """<b>"escaping"&nbsp;'forward range mark before'</b>{{\s+}}REQ-1
+CHECK-SRC-NEXT:
+ +# Forward range marker after. +CHECK-SRC: 
{{\s+}}<b>"escaping"&nbsp;'forward range mark after'</b>{{\s+}}/REQ-1
+CHECK-SRC-NEXT:
+ +# Range marker before. +CHECK-SRC: 
# <b>"escaping"&nbsp;'range mark before'</b> @sdoc[REQ-1]
+ +# Range marker after. +CHECK-SRC: 
# <b>"escaping"&nbsp;'range mark after'</b> @sdoc[/REQ-1]