From 0b0a43ce65e383d055da5b08aa7a151bf3145184 Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Thu, 23 Nov 2023 21:00:42 +0100 Subject: [PATCH 01/18] UI: Diff Screen to review changes between document trees --- drafts/requirements/strictdoc.toml | 1 + strictdoc.toml | 1 + strictdoc/backend/sdoc/models/requirement.py | 11 + strictdoc/backend/sdoc/reader.py | 15 +- strictdoc/core/project_config.py | 7 + .../html/templates/_shared/nav.jinja.html | 12 + .../html/templates/screens/git/form.jinja | 33 ++ .../screens/git/frame_project_tree.jinja | 44 +++ .../html/templates/screens/git/index.jinja | 68 ++++ .../html/templates/screens/git/main.jinja | 30 ++ .../html/templates/screens/git/node.jinja | 94 ++++++ .../templates/screens/git/requirement.jinja | 35 ++ strictdoc/git/git_client.py | 114 +++++++ strictdoc/git/project_diff_analyzer.py | 301 ++++++++++++++++++ strictdoc/helpers/md5.py | 15 + strictdoc/server/app.py | 2 + strictdoc/server/routers/other_router.py | 201 ++++++++++++ 17 files changed, 976 insertions(+), 8 deletions(-) create mode 100644 strictdoc/export/html/templates/screens/git/form.jinja create mode 100644 strictdoc/export/html/templates/screens/git/frame_project_tree.jinja create mode 100644 strictdoc/export/html/templates/screens/git/index.jinja create mode 100644 strictdoc/export/html/templates/screens/git/main.jinja create mode 100644 strictdoc/export/html/templates/screens/git/node.jinja create mode 100644 strictdoc/export/html/templates/screens/git/requirement.jinja create mode 100644 strictdoc/git/git_client.py create mode 100644 strictdoc/git/project_diff_analyzer.py create mode 100644 strictdoc/helpers/md5.py create mode 100644 strictdoc/server/routers/other_router.py diff --git a/drafts/requirements/strictdoc.toml b/drafts/requirements/strictdoc.toml index 3b5cb92e9..fdeabb46a 100644 --- a/drafts/requirements/strictdoc.toml +++ b/drafts/requirements/strictdoc.toml @@ -19,4 +19,5 @@ features = [ # "REQUIREMENTS_COVERAGE_SCREEN", # "REQUIREMENT_TO_SOURCE_TRACEABILITY", "HTML2PDF", + "DIFF", ] diff --git a/strictdoc.toml b/strictdoc.toml index 68da0f082..012b13ff3 100644 --- a/strictdoc.toml +++ b/strictdoc.toml @@ -8,6 +8,7 @@ features = [ "TRACEABILITY_SCREEN", "DEEP_TRACEABILITY_SCREEN", "SEARCH", + "DIFF", # Stable features but not used by StrictDoc. # "MATHJAX" diff --git a/strictdoc/backend/sdoc/models/requirement.py b/strictdoc/backend/sdoc/models/requirement.py index c2f19bf25..198a2cc91 100644 --- a/strictdoc/backend/sdoc/models/requirement.py +++ b/strictdoc/backend/sdoc/models/requirement.py @@ -396,6 +396,17 @@ def enumerate_fields(self): for requirement_field_list in requirement_fields: yield from requirement_field_list + def enumerate_all_fields(self): + for field in self.enumerate_fields(): + if field.field_name == "REFS": + continue + meta_field_value = ( + field.field_value + if field.field_value + else field.field_value_multiline + ) + yield field.field_name, meta_field_value + def enumerate_meta_fields( self, skip_single_lines=False, skip_multi_lines=False ): diff --git a/strictdoc/backend/sdoc/reader.py b/strictdoc/backend/sdoc/reader.py index 194f9398e..ab1100d5e 100644 --- a/strictdoc/backend/sdoc/reader.py +++ b/strictdoc/backend/sdoc/reader.py @@ -16,7 +16,7 @@ from strictdoc.backend.sdoc.processor import ParseContext, SDocParsingProcessor from strictdoc.helpers.cast import assert_cast from strictdoc.helpers.exception import StrictDocException -from strictdoc.helpers.file_modification_time import get_file_modification_time +from strictdoc.helpers.md5 import get_file_md5 from strictdoc.helpers.pickle import pickle_dump, pickle_load from strictdoc.helpers.textx import drop_textx_meta @@ -92,6 +92,8 @@ def read_from_file(self, file_path: str) -> Document: else os.path.abspath(file_path) ) + file_md5 = get_file_md5(file_path) + # File name contains an MD5 hash of its full path to ensure the # uniqueness of the cached items. Additionally, the unique file name # contains a full path to the output root to prevent collisions @@ -102,7 +104,7 @@ def read_from_file(self, file_path: str) -> Document: unique_identifier.encode("utf-8") ).hexdigest() file_name = os.path.basename(full_path_to_file) - file_name += "_" + unique_identifier_md5 + file_name += "_" + unique_identifier_md5 + "_" + file_md5 path_to_cached_file = os.path.join( path_to_tmp_dir, @@ -111,12 +113,9 @@ def read_from_file(self, file_path: str) -> Document: ) if os.path.isfile(path_to_cached_file): - cached_file_mtime = get_file_modification_time(path_to_cached_file) - sdoc_file_mtime = get_file_modification_time(file_path) - if sdoc_file_mtime < cached_file_mtime: - with open(path_to_cached_file, "rb") as cache_file: - sdoc_pickled = cache_file.read() - return assert_cast(pickle_load(sdoc_pickled), Document) + with open(path_to_cached_file, "rb") as cache_file: + sdoc_pickled = cache_file.read() + return assert_cast(pickle_load(sdoc_pickled), Document) path_to_cached_file_dir = os.path.dirname(path_to_cached_file) Path(path_to_cached_file_dir).mkdir(parents=True, exist_ok=True) diff --git a/strictdoc/core/project_config.py b/strictdoc/core/project_config.py index 6170a6b1c..3f351f938 100644 --- a/strictdoc/core/project_config.py +++ b/strictdoc/core/project_config.py @@ -31,6 +31,7 @@ class ProjectFeature(str, Enum): SEARCH = "SEARCH" HTML2PDF = "HTML2PDF" REQIF = "REQIF" + DIFF = "DIFF" PROJECT_STATISTICS_SCREEN = "PROJECT_STATISTICS_SCREEN" STANDALONE_DOCUMENT_SCREEN = "STANDALONE_DOCUMENT_SCREEN" REQUIREMENTS_COVERAGE_SCREEN = "REQUIREMENTS_COVERAGE_SCREEN" @@ -238,6 +239,12 @@ def is_activated_search(self): def is_activated_html2pdf(self) -> bool: return ProjectFeature.HTML2PDF in self.project_features + def is_activated_diff(self) -> bool: + return ( + self.is_running_on_server + and ProjectFeature.DIFF in self.project_features + ) + def is_activated_reqif(self) -> bool: return ProjectFeature.REQIF in self.project_features diff --git a/strictdoc/export/html/templates/_shared/nav.jinja.html b/strictdoc/export/html/templates/_shared/nav.jinja.html index 866314820..1522f4877 100644 --- a/strictdoc/export/html/templates/_shared/nav.jinja.html +++ b/strictdoc/export/html/templates/_shared/nav.jinja.html @@ -58,4 +58,16 @@ {%- endif -%} + {%- if project_config.is_activated_diff() -%} + + DIFF + +{%- endif -%} + diff --git a/strictdoc/export/html/templates/screens/git/form.jinja b/strictdoc/export/html/templates/screens/git/form.jinja new file mode 100644 index 000000000..6a47b805f --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/form.jinja @@ -0,0 +1,33 @@ + +
+ + + + + +
diff --git a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja new file mode 100644 index 000000000..006aad655 --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja @@ -0,0 +1,44 @@ + +{%- if not document_tree_iterator.is_empty_tree() -%} +
+ {%- for folder_or_file in document_tree_iterator.iterator_files_first(): -%} + {%- if folder_or_file.is_folder(): %} + {%- if folder_or_file.files|length > 0 %} +
{% include "_res/svg__separator.jinja.html" %}{{ folder_or_file.rel_path }}
+ {% endif %} + {% else %} + {%- set document_ = document_tree.get_document_by_path(folder_or_file.get_full_path()) %} + + {% set document_md5 = self_stats.get_md5_by_node(document_) %} + {% set document_modified = not other_stats.contains_document_md5(document_md5) %} + +
+ + + {{ document_.title }} + + + {% with node=document_ %} + {% include "screens/git/node.jinja" %} + {% endwith %} +
+ + {% endif %} + {%- endfor -%} +
+{%- else -%} + 🐛 The project has no documents yet. +{%- endif -%} + diff --git a/strictdoc/export/html/templates/screens/git/index.jinja b/strictdoc/export/html/templates/screens/git/index.jinja new file mode 100644 index 000000000..12293962b --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/index.jinja @@ -0,0 +1,68 @@ +{% extends "base.jinja.html" %} +{% set template_type = "Git" %} + +{% block head_css %} + {{ super() }} + + +{% endblock head_css %} + +{% block head_scripts %} + {{ super() }} + + {%- if project_config.is_running_on_server and not standalone -%} + + + {%- endif -%} +{% endblock head_scripts %} + +{% block title %} + {{ project_config.project_title }} - {{ template_type }} +{% endblock title %} + +{% block viewtype %}diff{% endblock viewtype %} + +{% block layout_nav %} + {% include "_shared/nav.jinja.html" %} +{% endblock layout_nav %} + +{% block tree_content %} + {# NOTHING #} +{% endblock tree_content %} + +{% block header_content %} + {%- with header__pagetype=template_type -%} + {% include "components/header/index.jinja" %} + {%- endwith -%} +{% endblock header_content %} + +{% block main_content %} + {% include "screens/git/main.jinja" %} +{% endblock main_content %} diff --git a/strictdoc/export/html/templates/screens/git/main.jinja b/strictdoc/export/html/templates/screens/git/main.jinja new file mode 100644 index 000000000..8ecdef86e --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/main.jinja @@ -0,0 +1,30 @@ +
+ {% include "screens/git/form.jinja" %} + + {% if results %} +
+
+ {% with + document_tree=document_tree_lhs, + document_tree_iterator=documents_iterator_lhs, + traceability_index=traceability_index_lhs, + self_stats=lhs_stats, + other_stats=rhs_stats + %} + {% include "screens/git/frame_project_tree.jinja" %} + {% endwith %} +
+
+ {% with + document_tree=document_tree_rhs, + document_tree_iterator=documents_iterator_rhs, + traceability_index=traceability_index_rhs, + self_stats=rhs_stats, + other_stats=lhs_stats + %} + {% include "screens/git/frame_project_tree.jinja" %} + {% endwith %} +
+
+ {% endif %} +
\ No newline at end of file diff --git a/strictdoc/export/html/templates/screens/git/node.jinja b/strictdoc/export/html/templates/screens/git/node.jinja new file mode 100644 index 000000000..e136fb889 --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/node.jinja @@ -0,0 +1,94 @@ + diff --git a/strictdoc/export/html/templates/screens/git/requirement.jinja b/strictdoc/export/html/templates/screens/git/requirement.jinja new file mode 100644 index 000000000..1df685b20 --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/requirement.jinja @@ -0,0 +1,35 @@ +
+ {% for meta_field in requirement.enumerate_all_fields() %} +
+ {{ meta_field[0] }}: {{ meta_field[1] }} +
+ {% endfor %} + + {%- if traceability_index.has_parent_requirements(requirement) %} +
Parent relations:
+
+ +
+ {%- endif %} + +
diff --git a/strictdoc/git/git_client.py b/strictdoc/git/git_client.py new file mode 100644 index 000000000..0efe95db5 --- /dev/null +++ b/strictdoc/git/git_client.py @@ -0,0 +1,114 @@ +import os.path +import subprocess +from typing import Optional + + +class GitClient: + def __init__(self, path_to_git_root: str): + assert os.path.isdir(path_to_git_root) + self.path_to_git_root: str = path_to_git_root + + def add_file(self, path_to_file): + result = subprocess.run( + ["git", "add", path_to_file], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0, result + + def add_all(self): + result = subprocess.run( + ["git", "add", "."], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0, result + + def commit(self, message: str): + result = subprocess.run( + ["git", "commit", "-m", message], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0, result + + def check_revision(self, revision: str): + assert isinstance(revision, str) + assert len(revision) > 0 + result = subprocess.run( + ["git", "rev-parse", revision], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + raise LookupError(f"Non-existing revision: {revision}") + + def commit_all(self, message: str): + result = subprocess.run( + ["git", "commit", "-a", "-m", message], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0, result + + def rebase_from_main(self): + result = subprocess.run( + ["git", "fetch", "origin"], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0, result + result = subprocess.run( + ["git", "rebase", "origin/main"], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0, result + + def push(self): + result = subprocess.run( + ["git", "push", "origin"], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + assert result.returncode == 0, result + + def hard_reset(self, revision: Optional[str] = None): + reset_args = ["git", "reset", "--hard"] + if revision is not None: + reset_args.append(revision) + result = subprocess.run( + reset_args, + cwd=self.path_to_git_root, + capture_output=False, + text=True, + check=True, + ) + assert result.returncode == 0, result + + def clean(self): + result = subprocess.run( + ["git", "clean", "-fd"], + cwd=self.path_to_git_root, + capture_output=False, + text=True, + check=True, + ) + assert result.returncode == 0, result diff --git a/strictdoc/git/project_diff_analyzer.py b/strictdoc/git/project_diff_analyzer.py new file mode 100644 index 000000000..2d3fdbf51 --- /dev/null +++ b/strictdoc/git/project_diff_analyzer.py @@ -0,0 +1,301 @@ +import hashlib +import statistics +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from typing import Any, Dict, List, Optional, Set + +from strictdoc.backend.sdoc.models.document import Document +from strictdoc.backend.sdoc.models.reference import ( + ParentReqReference, +) +from strictdoc.backend.sdoc.models.requirement import ( + Requirement, + RequirementField, +) +from strictdoc.backend.sdoc.models.section import Section +from strictdoc.core.document_iterator import DocumentCachingIterator +from strictdoc.core.traceability_index import TraceabilityIndex +from strictdoc.helpers.cast import assert_cast +from strictdoc.helpers.md5 import get_md5 + + +def similar(a, b): + return SequenceMatcher(None, a, b).ratio() + + +def calculate_similarity(lhs: Requirement, rhs: Requirement) -> float: + similar_fields = [] + for field_name_, field_values_ in lhs.ordered_fields_lookup.items(): + if field_name_ == "COMMENT": + continue + + if field_name_ not in rhs.ordered_fields_lookup: + continue + + if field_name_ not in ("TITLE", "STATEMENT", "RATIONALE"): + continue + + rhs_field_values = rhs.ordered_fields_lookup[field_name_] + + lhs_field_value = ( + field_values_[0].field_value + if field_values_[0].field_value is not None + else field_values_[0].field_value_multiline + ) + rhs_field_value = ( + rhs_field_values[0].field_value + if rhs_field_values[0].field_value is not None + else rhs_field_values[0].field_value_multiline + ) + + similar_fields.append(similar(lhs_field_value, rhs_field_value)) + + return statistics.mean(similar_fields) + + +@dataclass +class ProjectTreeDiffStats: + document_md5_hashes: Set[str] = field(default_factory=set) + requirement_md5_hashes: Set[str] = field(default_factory=set) + section_md5_hashes: Set[str] = field(default_factory=set) + free_text_md5_hashes: Set[str] = field(default_factory=set) + map_nodes_to_hashes: Dict[Any, str] = field(default_factory=dict) + map_uid_to_nodes: Dict[str, Any] = field(default_factory=dict) + map_titles_to_nodes: Dict[str, List] = field(default_factory=dict) + map_statements_to_nodes: Dict[str, Any] = field(default_factory=dict) + + cache_requirement_to_requirement: Dict[Requirement, Requirement] = field( + default_factory=dict + ) + + def get_md5_by_node(self, node) -> str: + return self.map_nodes_to_hashes[node] + + def contains_requirement_md5(self, requirement_md5: str) -> bool: + return requirement_md5 in self.requirement_md5_hashes + + def contains_free_text_md5(self, free_text_md5: str) -> bool: + return free_text_md5 in self.free_text_md5_hashes + + def contains_section_md5(self, section_md5: str) -> bool: + return section_md5 in self.section_md5_hashes + + def contains_document_md5(self, document_md5: str) -> bool: + return document_md5 in self.document_md5_hashes + + def contains_requirement_field( + self, requirement: Requirement, field_name: str, field_value: str + ): + assert isinstance(field_value, str) + other_requirement: Optional[Requirement] = self._find_requirement( + requirement + ) + if other_requirement is None: + return False + + if field_name not in other_requirement.ordered_fields_lookup: + return False + + other_requirement_fields = other_requirement.ordered_fields_lookup[ + field_name + ] + for field_ in other_requirement_fields: + if field_.field_value == field_value: + return True + if field_.field_value_multiline == field_value: + return True + return False + + def contains_requirement_relations( + self, + requirement: Requirement, + relation_uid: str, + relation_role: Optional[str], + ): + other_requirement: Optional[Requirement] = self._find_requirement( + requirement + ) + if other_requirement is None: + return False + for reference_ in other_requirement.references: + if isinstance(reference_, ParentReqReference): + parent_reference: ParentReqReference = assert_cast( + reference_, ParentReqReference + ) + if ( + parent_reference.ref_uid == relation_uid + and parent_reference.role == relation_role + ): + return True + return False + + def _find_requirement( + self, requirement: Requirement + ) -> Optional[Requirement]: + if requirement in self.cache_requirement_to_requirement: + return self.cache_requirement_to_requirement[requirement] + + requirement_parent_uids = set() + for parent_ in requirement.references: + if isinstance(parent_, ParentReqReference): + requirement_parent_uids.add(parent_.ref_uid) + + if ( + requirement.reserved_uid is None + or requirement.reserved_uid not in self.map_uid_to_nodes + ): + candidate_requirements: Dict[Requirement, float] = {} + + if ( + requirement.reserved_title is not None + and requirement.reserved_title in self.map_titles_to_nodes + ): + other_requirements = self.map_titles_to_nodes[ + requirement.reserved_title + ] + for other_requirement_ in other_requirements: + if ( + not other_requirement_.reserved_uid + in requirement_parent_uids + ): + candidate_requirements[other_requirement_] = 0 + + for candidate_requirement_ in candidate_requirements.keys(): + candidate_requirements[ + candidate_requirement_ + ] = calculate_similarity(requirement, candidate_requirement_) + + if len(candidate_requirements) > 0: + candidate_requirement = max( + candidate_requirements, key=candidate_requirements.get + ) + self.cache_requirement_to_requirement[ + requirement + ] = candidate_requirement + return candidate_requirement + return None + + other_requirement: Requirement = self.map_uid_to_nodes[ + requirement.reserved_uid + ] + return other_requirement + + +class ProjectDiffAnalyzer: + @staticmethod + def analyze_document_tree( + traceability_index: TraceabilityIndex + ) -> ProjectTreeDiffStats: + document_tree_stats: ProjectTreeDiffStats = ProjectTreeDiffStats() + + for document in traceability_index.document_tree.document_list: + ProjectDiffAnalyzer.analyze_document(document, document_tree_stats) + + return document_tree_stats + + @staticmethod + def analyze_document( + document: Document, + document_tree_stats: ProjectTreeDiffStats, + ) -> None: + document_iterator = DocumentCachingIterator(document) + + map_nodes_to_hashers: Dict[Any, Any] = {document: hashlib.md5()} + + # Document's top level free text. + if len(document.free_texts) > 0: + free_text = document.free_texts[0] + free_text_text = document.free_texts[0].get_parts_as_text() + free_text_md5 = get_md5(free_text_text) + document_tree_stats.free_text_md5_hashes.add(free_text_md5) + document_tree_stats.map_nodes_to_hashes[free_text] = free_text_md5 + + for node in document_iterator.all_content(): + if isinstance(node, Section): + hasher = hashlib.md5() + hasher.update(node.title.encode("utf-8")) + if len(node.free_texts) > 0: + free_text = node.free_texts[0] + free_text_text = node.free_texts[0].get_parts_as_text() + free_text_md5 = get_md5(free_text_text) + document_tree_stats.free_text_md5_hashes.add(free_text_md5) + document_tree_stats.map_nodes_to_hashes[ + free_text + ] = free_text_md5 + + hasher.update(free_text_text.encode("utf-8")) + map_nodes_to_hashers[node] = hasher + + elif isinstance(node, Requirement): + if node.reserved_uid is not None: + document_tree_stats.map_uid_to_nodes[ + node.reserved_uid + ] = node + + hasher = hashlib.md5() + for ( + field_name_, + field_values_, + ) in node.ordered_fields_lookup.items(): + requirement_field: RequirementField = field_values_[0] + if field_name_ == "TITLE": + this_title_requirements = ( + document_tree_stats.map_titles_to_nodes.setdefault( + requirement_field.field_value, [] + ) + ) + this_title_requirements.append(node) + elif field_name_ == "STATEMENT": + statement_value: str = assert_cast( + requirement_field.field_value + if requirement_field.field_value is not None + else requirement_field.field_value_multiline, + str, + ) + document_tree_stats.map_statements_to_nodes[ + statement_value + ] = node + + if requirement_field.field_value is not None: + hasher.update( + requirement_field.field_value.encode("utf-8") + ) + elif requirement_field.field_value_multiline is not None: + hasher.update( + requirement_field.field_value_multiline.encode( + "utf-8" + ) + ) + else: + # WIP + continue + map_nodes_to_hashers[node] = hasher + else: + raise AssertionError + + def recurse(node): + assert isinstance(node, (Section, Document)) + for subnode in node.section_contents: + if isinstance(subnode, Section): + map_nodes_to_hashers[node].update(recurse(subnode)) + elif isinstance(subnode, Requirement): + node_md5 = ( + map_nodes_to_hashers[subnode] + .hexdigest() + .encode("utf-8") + ) + map_nodes_to_hashers[node].update(node_md5) + return map_nodes_to_hashers[node].hexdigest().encode("utf-8") + + recurse(document) + + for node_, node_hasher_ in map_nodes_to_hashers.items(): + node_md5 = node_hasher_.hexdigest() + document_tree_stats.map_nodes_to_hashes[node_] = node_md5 + + if isinstance(node_, Section): + document_tree_stats.section_md5_hashes.add(node_md5) + elif isinstance(node_, Requirement): + document_tree_stats.requirement_md5_hashes.add(node_md5) + elif isinstance(node_, Document): + document_tree_stats.document_md5_hashes.add(node_md5) diff --git a/strictdoc/helpers/md5.py b/strictdoc/helpers/md5.py new file mode 100644 index 000000000..0a436cb73 --- /dev/null +++ b/strictdoc/helpers/md5.py @@ -0,0 +1,15 @@ +import hashlib + + +def get_md5(obj: str): + return hashlib.md5(obj.encode("utf-8")).hexdigest() + + +def get_file_md5(path, buf_size=65536): + m = hashlib.md5() + with open(path, "rb") as f: + b = f.read(buf_size) + while len(b) > 0: + m.update(b) + b = f.read(buf_size) + return m.hexdigest() diff --git a/strictdoc/server/app.py b/strictdoc/server/app.py index ead4ce199..7de24bc72 100644 --- a/strictdoc/server/app.py +++ b/strictdoc/server/app.py @@ -10,6 +10,7 @@ from strictdoc.helpers.pickle import pickle_load from strictdoc.server.config import SDocServerEnvVariable from strictdoc.server.routers.main_router import create_main_router +from strictdoc.server.routers.other_router import create_other_router def create_app( @@ -44,6 +45,7 @@ async def add_process_time_header( # pylint: disable=unused-variable allow_headers=["*"], ) + app.include_router(create_other_router(project_config=project_config)) app.include_router( create_main_router( server_config=server_config, project_config=project_config diff --git a/strictdoc/server/routers/other_router.py b/strictdoc/server/routers/other_router.py new file mode 100644 index 000000000..920f55da8 --- /dev/null +++ b/strictdoc/server/routers/other_router.py @@ -0,0 +1,201 @@ +import os +import subprocess +from copy import deepcopy +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter +from starlette.responses import HTMLResponse, Response + +from strictdoc import __version__ +from strictdoc.core.document_tree_iterator import DocumentTreeIterator +from strictdoc.core.project_config import ProjectConfig +from strictdoc.core.traceability_index import TraceabilityIndex +from strictdoc.core.traceability_index_builder import TraceabilityIndexBuilder +from strictdoc.export.html.document_type import DocumentType +from strictdoc.export.html.html_templates import HTMLTemplates +from strictdoc.export.html.renderers.link_renderer import LinkRenderer +from strictdoc.git.git_client import GitClient +from strictdoc.git.project_diff_analyzer import ( + ProjectDiffAnalyzer, + ProjectTreeDiffStats, +) +from strictdoc.helpers.parallelizer import NullParallelizer +from strictdoc.helpers.timing import measure_performance +from strictdoc.server.routers.main_router import HTTP_STATUS_PRECONDITION_FAILED + + +def create_other_router(project_config: ProjectConfig) -> APIRouter: + router = APIRouter() + + html_templates = HTMLTemplates.create( + project_config=project_config, + enable_caching=False, + strictdoc_last_update=datetime.today(), + ) + + @router.get("/diff") + def get_git_diff( + left_revision: Optional[str] = None, + right_revision: Optional[str] = None, + ): + if not project_config.is_activated_diff(): + return Response( + content="The DIFF feature is not activated in the project config.", + status_code=HTTP_STATUS_PRECONDITION_FAILED, + ) + left_revision_resolved = None + right_revision_resolved = None + results = False + + if left_revision is not None: + assert right_revision is not None + + git_client = GitClient(".") + try: + if left_revision != "HEAD+": + left_revision_resolved = git_client.check_revision( + left_revision + ) + else: + return HTMLResponse( + content=( + "Left revision argument 'HEAD+' is not supported. " + "'HEAD+' can only be used as a right revision argument." + ), + status_code=412, + ) + + if right_revision != "HEAD+": + right_revision_resolved = git_client.check_revision( + right_revision + ) + else: + right_revision_resolved = "HEAD+" + except LookupError as exception_: + return HTMLResponse(content=exception_.args[0], status_code=404) + results = True + + template = html_templates.jinja_environment().get_template( + "screens/git/index.jinja" + ) + + link_renderer = LinkRenderer( + root_path="", static_path=project_config.dir_for_sdoc_assets + ) + if not results: + output = template.render( + project_config=project_config, + document_type=DocumentType.document(), + link_document_type=DocumentType.document(), + standalone=False, + strictdoc_version=__version__, + link_renderer=link_renderer, + results=False, + left_revision=None, + right_revision=None, + ) + return HTMLResponse(content=output, status_code=200) + + assert left_revision_resolved is not None + assert right_revision_resolved is not None + + path_to_cwd = os.getcwd() + path_to_sandbox_dir = "/tmp/sandbox" + + with measure_performance("RSYNC"): + result = subprocess.run( + [ + "rsync", + # "-vvraP", + "-r", + "--partial", + "--delete", + "--exclude=build/", + "--exclude=__pycache__/", + "--exclude=.ruff_cache/", + "--exclude=output/", + "--exclude=strictdoc-project.github.io/", + ".", + path_to_sandbox_dir, + ], + cwd=path_to_cwd, + capture_output=False, + text=True, + check=True, + ) + assert result.returncode == 0, result + + git_client = GitClient(path_to_sandbox_dir) + if right_revision_resolved != "HEAD+": + git_client.hard_reset(revision=right_revision_resolved) + git_client.clean() + + parallelizer = NullParallelizer() + + project_config_copy: ProjectConfig = deepcopy(project_config) + assert project_config_copy.export_input_paths is not None + export_input_rel_path = os.path.relpath( + project_config_copy.export_input_paths[0], os.getcwd() + ) + export_input_abs_path = os.path.join( + path_to_sandbox_dir, export_input_rel_path + ) + project_config_copy.export_input_paths = [export_input_abs_path] + + traceability_index_rhs: TraceabilityIndex = ( + TraceabilityIndexBuilder.create( + project_config=project_config_copy, + parallelizer=parallelizer, + ) + ) + + if left_revision_resolved != "HEAD+": + git_client.hard_reset(revision=left_revision_resolved) + git_client.clean() + + traceability_index_lhs: TraceabilityIndex = ( + TraceabilityIndexBuilder.create( + project_config=project_config_copy, + parallelizer=parallelizer, + ) + ) + + lhs_stats: ProjectTreeDiffStats = ( + ProjectDiffAnalyzer.analyze_document_tree(traceability_index_lhs) + ) + rhs_stats: ProjectTreeDiffStats = ( + ProjectDiffAnalyzer.analyze_document_tree(traceability_index_rhs) + ) + + documents_iterator_lhs = DocumentTreeIterator( + traceability_index_lhs.document_tree + ) + documents_iterator_rhs = DocumentTreeIterator( + traceability_index_rhs.document_tree + ) + output = template.render( + project_config=project_config, + document_tree_lhs=traceability_index_lhs.document_tree, + document_tree_rhs=traceability_index_rhs.document_tree, + documents_iterator_lhs=documents_iterator_lhs, + documents_iterator_rhs=documents_iterator_rhs, + left_revision=left_revision, + right_revision=right_revision, + lhs_stats=lhs_stats, + rhs_stats=rhs_stats, + traceability_index_lhs=traceability_index_lhs, + traceability_index_rhs=traceability_index_rhs, + link_renderer=link_renderer, + document_type=DocumentType.document(), + link_document_type=DocumentType.document(), + standalone=False, + strictdoc_version=__version__, + results=True, + ) + return HTMLResponse( + content=output, + status_code=200, + ) + + return router From db8c73ce4b14794ae3a5b34ef419c03dc66a58c7 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Fri, 1 Dec 2023 20:25:39 +0100 Subject: [PATCH 02/18] UI: diff icon in main navigation bar --- strictdoc/export/html/_static/element.css | 1 + strictdoc/export/html/templates/_res/svg_ico16_diff.jinja | 8 ++++++++ strictdoc/export/html/templates/_shared/nav.jinja.html | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 strictdoc/export/html/templates/_res/svg_ico16_diff.jinja diff --git a/strictdoc/export/html/_static/element.css b/strictdoc/export/html/_static/element.css index 687714358..f1a6a1d32 100644 --- a/strictdoc/export/html/_static/element.css +++ b/strictdoc/export/html/_static/element.css @@ -505,6 +505,7 @@ sdoc-anchor[visible]:hover .anchor_back_links_number { /* should be like .nav .nav_button:hover */ [data-viewtype="document-tree"] [data-link="index"], [data-viewtype="search"] [data-link="search"], +[data-viewtype="diff"] [data-link="diff"], [data-viewtype="requirements-coverage"] [data-link="requirements_coverage"], [data-viewtype="coverage-tree"] [data-link="source_coverage"] { color: var(--color-bg-contrast); diff --git a/strictdoc/export/html/templates/_res/svg_ico16_diff.jinja b/strictdoc/export/html/templates/_res/svg_ico16_diff.jinja new file mode 100644 index 000000000..da67e3d1b --- /dev/null +++ b/strictdoc/export/html/templates/_res/svg_ico16_diff.jinja @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/strictdoc/export/html/templates/_shared/nav.jinja.html b/strictdoc/export/html/templates/_shared/nav.jinja.html index 1522f4877..af1cd61c5 100644 --- a/strictdoc/export/html/templates/_shared/nav.jinja.html +++ b/strictdoc/export/html/templates/_shared/nav.jinja.html @@ -66,7 +66,7 @@ title="Diff" data-testid="project-tree-link-diff" > - DIFF + {% include "_res/svg_ico16_diff.jinja" %} {%- endif -%} From bb513a069db9c9ad59b4b5ea230caf69d396e77c Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Fri, 1 Dec 2023 21:28:33 +0100 Subject: [PATCH 03/18] export/html: escape free text to avoid html markup issues --- strictdoc/backend/sdoc/models/free_text.py | 3 ++- strictdoc/export/html/templates/screens/git/form.jinja | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/strictdoc/backend/sdoc/models/free_text.py b/strictdoc/backend/sdoc/models/free_text.py index 494e9c266..d01f06771 100644 --- a/strictdoc/backend/sdoc/models/free_text.py +++ b/strictdoc/backend/sdoc/models/free_text.py @@ -1,3 +1,4 @@ +import html from typing import List from strictdoc.backend.sdoc.models.anchor import Anchor @@ -42,7 +43,7 @@ def get_parts_as_text(self) -> str: text += "\n" else: raise NotImplementedError(part) - return text + return html.escape(text) class FreeTextContainer(FreeText): diff --git a/strictdoc/export/html/templates/screens/git/form.jinja b/strictdoc/export/html/templates/screens/git/form.jinja index 6a47b805f..cfa7cc980 100644 --- a/strictdoc/export/html/templates/screens/git/form.jinja +++ b/strictdoc/export/html/templates/screens/git/form.jinja @@ -29,5 +29,5 @@ name="right_revision" /> - + From a6c91f07f71f1a5aaee5e1a0bb5ba58a68ead119 Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Sat, 2 Dec 2023 22:03:25 +0100 Subject: [PATCH 04/18] export/html: escape free text to avoid html markup issues (dedicated method) --- strictdoc/backend/sdoc/models/free_text.py | 5 ++++- strictdoc/export/html/templates/screens/git/node.jinja | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/strictdoc/backend/sdoc/models/free_text.py b/strictdoc/backend/sdoc/models/free_text.py index d01f06771..cff818762 100644 --- a/strictdoc/backend/sdoc/models/free_text.py +++ b/strictdoc/backend/sdoc/models/free_text.py @@ -43,7 +43,10 @@ def get_parts_as_text(self) -> str: text += "\n" else: raise NotImplementedError(part) - return html.escape(text) + return text + + def get_parts_as_text_escaped(self) -> str: + return html.escape(self.get_parts_as_text()) class FreeTextContainer(FreeText): diff --git a/strictdoc/export/html/templates/screens/git/node.jinja b/strictdoc/export/html/templates/screens/git/node.jinja index e136fb889..71c459012 100644 --- a/strictdoc/export/html/templates/screens/git/node.jinja +++ b/strictdoc/export/html/templates/screens/git/node.jinja @@ -25,7 +25,7 @@ style="background-color: #E4F1C9;" {% endif %} > - {{ free_text.get_parts_as_text() }} + {{ free_text.get_parts_as_text_escaped() }} From 097766e6e18d375fdde788e09be653a284cf5792 Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Sat, 2 Dec 2023 22:38:18 +0100 Subject: [PATCH 05/18] UI: Diff Screen: step towards a better red-green color coding of changes --- .../screens/git/frame_project_tree.jinja | 2 +- .../html/templates/screens/git/main.jinja | 6 ++- .../html/templates/screens/git/node.jinja | 4 +- .../templates/screens/git/requirement.jinja | 19 +++++-- strictdoc/git/project_diff_analyzer.py | 50 +++++++++++++++++-- strictdoc/helpers/diff.py | 34 +++++++++++++ 6 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 strictdoc/helpers/diff.py diff --git a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja index 006aad655..aa8f4031d 100644 --- a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja +++ b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja @@ -24,7 +24,7 @@ {{ document_.title }} diff --git a/strictdoc/export/html/templates/screens/git/main.jinja b/strictdoc/export/html/templates/screens/git/main.jinja index 8ecdef86e..5eabe97ee 100644 --- a/strictdoc/export/html/templates/screens/git/main.jinja +++ b/strictdoc/export/html/templates/screens/git/main.jinja @@ -9,7 +9,8 @@ document_tree_iterator=documents_iterator_lhs, traceability_index=traceability_index_lhs, self_stats=lhs_stats, - other_stats=rhs_stats + other_stats=rhs_stats, + side="left" %} {% include "screens/git/frame_project_tree.jinja" %} {% endwith %} @@ -20,7 +21,8 @@ document_tree_iterator=documents_iterator_rhs, traceability_index=traceability_index_rhs, self_stats=rhs_stats, - other_stats=lhs_stats + other_stats=lhs_stats, + side="right" %} {% include "screens/git/frame_project_tree.jinja" %} {% endwith %} diff --git a/strictdoc/export/html/templates/screens/git/node.jinja b/strictdoc/export/html/templates/screens/git/node.jinja index 71c459012..c2eafa32b 100644 --- a/strictdoc/export/html/templates/screens/git/node.jinja +++ b/strictdoc/export/html/templates/screens/git/node.jinja @@ -46,7 +46,7 @@ @@ -72,7 +72,7 @@ diff --git a/strictdoc/export/html/templates/screens/git/requirement.jinja b/strictdoc/export/html/templates/screens/git/requirement.jinja index 1df685b20..d5d7f1d16 100644 --- a/strictdoc/export/html/templates/screens/git/requirement.jinja +++ b/strictdoc/export/html/templates/screens/git/requirement.jinja @@ -1,11 +1,20 @@
{% for meta_field in requirement.enumerate_all_fields() %} + {% set field_modified = requirement_modified and not other_stats.contains_requirement_field(requirement, meta_field[0], meta_field[1]) %}
+ {% if field_modified %} + {{ other_stats.get_diffed_requirement_field(requirement, meta_field[0], meta_field[1], side) }} + {% else %} {{ meta_field[0] }}: {{ meta_field[1] }} + {% endif %}
{% endfor %} @@ -16,7 +25,11 @@ {%- for parent_requirement_, relation_role_ in traceability_index.get_parent_relations_with_roles(requirement) %}
  • diff --git a/strictdoc/git/project_diff_analyzer.py b/strictdoc/git/project_diff_analyzer.py index 2d3fdbf51..f21295a07 100644 --- a/strictdoc/git/project_diff_analyzer.py +++ b/strictdoc/git/project_diff_analyzer.py @@ -1,7 +1,6 @@ import hashlib import statistics from dataclasses import dataclass, field -from difflib import SequenceMatcher from typing import Any, Dict, List, Optional, Set from strictdoc.backend.sdoc.models.document import Document @@ -16,13 +15,10 @@ from strictdoc.core.document_iterator import DocumentCachingIterator from strictdoc.core.traceability_index import TraceabilityIndex from strictdoc.helpers.cast import assert_cast +from strictdoc.helpers.diff import get_colored_diff_string, similar from strictdoc.helpers.md5 import get_md5 -def similar(a, b): - return SequenceMatcher(None, a, b).ratio() - - def calculate_similarity(lhs: Requirement, rhs: Requirement) -> float: similar_fields = [] for field_name_, field_values_ in lhs.ordered_fields_lookup.items(): @@ -106,6 +102,50 @@ def contains_requirement_field( return True return False + def get_diffed_requirement_field( + self, + requirement: Requirement, + field_name: str, + field_value: str, + side: str, + ): + assert isinstance(field_value, str) + assert side in ("left", "right") + + other_requirement: Optional[Requirement] = self._find_requirement( + requirement + ) + if ( + other_requirement is None + or field_name not in other_requirement.ordered_fields_lookup + ): + return field_name + ": " + field_value + + other_requirement_fields = other_requirement.ordered_fields_lookup[ + field_name + ] + + other_field_value = None + for field_ in other_requirement_fields: + if field_.field_value is not None: + other_field_value = field_.field_value + break + if field_.field_value_multiline is not None: + other_field_value = field_.field_value_multiline + break + assert other_field_value is not None + + if side == "left": + colored_field_value = get_colored_diff_string( + field_value, other_field_value, side + ) + return field_name + ": " + colored_field_value + else: + colored_field_value = get_colored_diff_string( + other_field_value, field_value, side + ) + return field_name + ": " + colored_field_value + def contains_requirement_relations( self, requirement: Requirement, diff --git a/strictdoc/helpers/diff.py b/strictdoc/helpers/diff.py new file mode 100644 index 000000000..225f2eff2 --- /dev/null +++ b/strictdoc/helpers/diff.py @@ -0,0 +1,34 @@ +import difflib +from difflib import SequenceMatcher + + +def similar(a, b): + return SequenceMatcher(None, a, b).ratio() + + +red = lambda text: f'{text}' +green = lambda text: f'{text}' +blue = lambda text: f'{text}' +white = lambda text: f'{text}' + + +def get_colored_diff_string(old, new, flag: str): + assert flag in ("left", "right") + + result = "" + codes = difflib.SequenceMatcher(a=old, b=new).get_opcodes() + for code in codes: + if code[0] == "equal": + result += white(old[code[1] : code[2]]) + elif code[0] == "delete": + if flag == "left": + result += red(old[code[1] : code[2]]) + elif code[0] == "insert": + if flag == "right": + result += green(new[code[3] : code[4]]) + elif code[0] == "replace": + if flag == "left": + result += red(old[code[1] : code[2]]) + else: + result += green(new[code[3] : code[4]]) + return result From bc9f40aa5bed646585b1a5cf56aded708b52fde9 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Fri, 1 Dec 2023 23:13:53 +0100 Subject: [PATCH 06/18] export/html: components: add button/diff.jinja --- .../html/templates/components/button/diff.jinja | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 strictdoc/export/html/templates/components/button/diff.jinja diff --git a/strictdoc/export/html/templates/components/button/diff.jinja b/strictdoc/export/html/templates/components/button/diff.jinja new file mode 100644 index 000000000..78a1fd833 --- /dev/null +++ b/strictdoc/export/html/templates/components/button/diff.jinja @@ -0,0 +1,15 @@ +{# + {{ form }} + Defined in the parent modal/form template. +#} + +{# {{ submit_name|default('Search') }} #} From 4b3061515170e1b3a1cefa29a34d6836cc828ea5 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Fri, 1 Dec 2023 23:14:28 +0100 Subject: [PATCH 07/18] export/html: update form CSS; remove 'sdoc-search' tags --- strictdoc/export/html/_static/form.css | 184 ++++++++++-------- .../templates/components/button/diff.jinja | 1 + .../templates/components/form/search.jinja | 7 +- .../html/templates/screens/git/form.jinja | 51 +++-- tests/integration/html_markup_validator.py | 1 - 5 files changed, 131 insertions(+), 113 deletions(-) diff --git a/strictdoc/export/html/_static/form.css b/strictdoc/export/html/_static/form.css index b6f56f229..d04edc0c0 100644 --- a/strictdoc/export/html/_static/form.css +++ b/strictdoc/export/html/_static/form.css @@ -91,87 +91,6 @@ sdoc-modal-footer { column-gap: var(--base-rhythm); } -/* sdoc-search */ - -sdoc-search { - display: block; - position: sticky; - top: 0; - z-index: 11; - - margin-bottom: calc(var(--base-rhythm)*4); - background-color: var(--color-bg-main); -} - -sdoc-search[success] { - box-shadow: 0 0 1rem 0.5rem var(--thumbBG); -} - -sdoc-search::before { - content: ''; - position: absolute; - bottom: 0; - top: -60px; - left: -40px; - right: -40px; - background-color: var(--color-bg-main); - z-index: 0; -} - -sdoc-search-grid { - display: grid; - place-items: stretch stretch; - place-content: stretch stretch; - grid-template-columns: minmax(0, 1fr) /* issue#1370 https://css-tricks.com/preventing-a-grid-blowout/ */ - minmax(0, min-content); - gap: - calc(var(--base-rhythm)*2) - var(--base-rhythm); - - position: relative; - padding: var(--base-rhythm); - background-color: var(--color-bg-contrast); - border-radius: 4px; - border: 1px solid var(--color-border); -} - -sdoc-search input[type="text"] { - padding: var(--base-rhythm); - font-size: var(--font-size); - border: 1px solid var(--color-border); - border-radius: 3px; - outline: transparent; - width: 100%; - transition: border-color calc(var(--transition, 0.2) * 1s) ease; -} - -sdoc-search input[type="text"]:focus { - border-color: var(--color-action); - color: var(--color-action); -} - -.sdoc-search-error, -.sdoc-search-success, -.sdoc-search-reset { - display: block; - padding: var(--base-rhythm) calc(var(--base-rhythm)*2); - position: relative; -} - -.sdoc-search-error { - color: var(--color-danger); -} - -.sdoc-search-success { - color: var(--color-action); -} - -.sdoc-search-reset { - position: absolute; - right: 0; - bottom: 0; -} - /* sdoc-form */ sdoc-form { @@ -661,3 +580,106 @@ form[data-controller~="scroll_into_view"] { scroll-snap-margin-top: calc(var(--base-padding) + 1px); scroll-margin-top: calc(var(--base-padding) + 1px); } + +/* input */ + +sdoc-form input[type="text"] { + padding: var(--base-rhythm); + font-size: var(--font-size); + border: 1px solid var(--color-border); + border-radius: 3px; + outline: transparent; + width: 100%; + transition: border-color calc(var(--transition, 0.2) * 1s) ease; +} + +sdoc-form input[type="text"]:focus { + border-color: var(--color-action); + color: var(--color-action); +} + +/* diff */ + +sdoc-form[diff] { + grid-column: 1 / -1; + + margin: 0; + background-color: transparent; + border: none; +} + +sdoc-form[diff] form { + display: flex; + gap: var(--base-rhythm); + position: relative; + padding: var(--base-rhythm); + background-color: var(--color-bg-contrast); + border: 1px solid var(--color-border); + border-radius: 4px; +} + +/* search */ + +sdoc-form[search] { + display: block; + position: sticky; + top: 0; + left: 0; + z-index: 11; + + margin-bottom: calc(var(--base-rhythm)*4); + background-color: transparent; + border: none; +} + +sdoc-form[search] form { + display: flex; + gap: var(--base-rhythm); + position: relative; + padding: var(--base-rhythm); + background-color: var(--color-bg-contrast); + border: 1px solid var(--color-border); + border-radius: 4px; +} + +sdoc-form[search][success] { + box-shadow: 0 0 1rem 0.5rem var(--thumbBG); +} + +sdoc-form[search]::before { + content: ''; + position: absolute; + bottom: 0; + top: -60px; + left: -60px; + right: -60px; + background-color: var(--color-bg-main); + z-index: 0; +} + +.sdoc-search-error, +.sdoc-search-success, +.sdoc-search-reset { + display: block; + padding: var(--base-rhythm) calc(var(--base-rhythm)*2); + position: relative; +} + +.sdoc-search-error { + color: var(--color-danger); +} + +.sdoc-search-success { + color: var(--color-action); +} + +.sdoc-search-reset { + position: absolute; + right: 0; + bottom: 0; +} + +.sdoc-search-error, +.sdoc-search-success { + padding-right: 120px; /* == sdoc-search-reset width */ +} diff --git a/strictdoc/export/html/templates/components/button/diff.jinja b/strictdoc/export/html/templates/components/button/diff.jinja index 78a1fd833..308f630bd 100644 --- a/strictdoc/export/html/templates/components/button/diff.jinja +++ b/strictdoc/export/html/templates/components/button/diff.jinja @@ -7,6 +7,7 @@ form="{{ form }}" {% endif %} data-turbo="true" + title="See the diff" class="action_icon" type="submit" data-action-type="submit" diff --git a/strictdoc/export/html/templates/components/form/search.jinja b/strictdoc/export/html/templates/components/form/search.jinja index 325dc6cb1..c483f4a04 100644 --- a/strictdoc/export/html/templates/components/form/search.jinja +++ b/strictdoc/export/html/templates/components/form/search.jinja @@ -1,4 +1,4 @@ - 0) %}success{% endif %} > @@ -6,17 +6,14 @@ action="/search" method="GET" > - {% include "components/button/search.jinja" %} - {%- if search_value|length > 0 -%} @@ -31,4 +28,4 @@ {%- endif -%} Clear query {%- endif -%} - + diff --git a/strictdoc/export/html/templates/screens/git/form.jinja b/strictdoc/export/html/templates/screens/git/form.jinja index cfa7cc980..23c47f129 100644 --- a/strictdoc/export/html/templates/screens/git/form.jinja +++ b/strictdoc/export/html/templates/screens/git/form.jinja @@ -1,33 +1,32 @@ - +
    - - + + {% include "components/button/diff.jinja" %} + -
    +     diff --git a/tests/integration/html_markup_validator.py b/tests/integration/html_markup_validator.py index 611c2876c..4d0237e0d 100644 --- a/tests/integration/html_markup_validator.py +++ b/tests/integration/html_markup_validator.py @@ -59,7 +59,6 @@ "sdoc-section-title, sdoc-section-text, " "sdoc-main-placeholder, " "sdoc-meta, sdoc-meta-label, sdoc-meta-field, " - "sdoc-search, sdoc-search-grid, " "sdoc-form, sdoc-form-grid, " "sdoc-form-header, sdoc-form-descr, sdoc-form-footer, " "sdoc-form-row, sdoc-form-row-main, sdoc-form-row-aside, " From b37d2d0502c614ba5542d86a5024c61a17c52d31 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sat, 2 Dec 2023 23:35:22 +0100 Subject: [PATCH 08/18] export/html: WIP: show diff in two columns --- strictdoc/export/html/_static/content.css | 102 ++++++++++++++++++ .../screens/git/frame_project_tree.jinja | 10 +- .../html/templates/screens/git/index.jinja | 24 ----- .../html/templates/screens/git/main.jinja | 13 ++- .../html/templates/screens/git/node.jinja | 12 +-- 5 files changed, 122 insertions(+), 39 deletions(-) diff --git a/strictdoc/export/html/_static/content.css b/strictdoc/export/html/_static/content.css index bd430d493..dc5cc247b 100644 --- a/strictdoc/export/html/_static/content.css +++ b/strictdoc/export/html/_static/content.css @@ -11,6 +11,25 @@ /* redefine main layout grid */ +[data-viewtype="diff"] .main { + padding: var(--base-gap); + padding-bottom: 0; + position: relative; + overflow: hidden; + scroll-behavior: smooth; + height: 100%; + width: 100%; + background-color: var(--color-bg-main); + + display: grid; + place-items: stretch stretch; + place-content: stretch stretch; + grid-template-columns: minmax(0, 1fr) /* issue#1370 https://css-tricks.com/preventing-a-grid-blowout/ */ + minmax(0, 1fr); + grid-template-rows: minmax(0, max-content) minmax(0, 1fr); + gap: var(--base-rhythm); +} + [data-viewtype="source-file"] .main { padding: 0; background: white; @@ -152,3 +171,86 @@ sdoc-main-placeholder.page-tips { color: unset; opacity: 0.5; } + +/* diff */ + +.diff { + overflow: auto; + overflow-y: scroll; + overflow-wrap: break-word; + /* border: 1px solid var(--thumbBG); */ + border-radius: 4px; + border: 1px solid var(--color-border); +} + +.diff[left] { + direction:rtl; +} + +.diff[right] { + direction:ltr; +} + +.diff_inner { + direction: initial; +} + +.diff_content { + display: flex; + flex-direction: column; + gap: var(--base-rhythm); + padding: var(--base-rhythm); +} + +.diff_folder { + display: flex; + align-items: center; + justify-content: flex-start; + column-gap: calc(var(--base-rhythm)/2); + font-size: var(--font-size-sm); + min-width: 0; + line-height: 1.5; + padding-top: var(--base-rhythm); +} + +.diff_document { + background-color: white; + border-radius: 4px; + padding: var(--base-rhythm); +} + +.diff_document > summary { + line-height: 1.2; +} + +.diff_document > summary:hover { + color: var(--color-fg-accent); +} + +.diff details.diff_document > summary::before, +.diff details.diff_document[open] > summary::before { + content: none; +} + +.diff details { + width: 100%; +} + +.diff summary { + list-style: none; + display: flex; + gap: var(--base-rhythm); + cursor: pointer; +} + +.diff summary::-webkit-details-marker { + display: none; +} + +.diff details > summary::before { + content:"+"; +} + +.diff details[open] > summary::before { + content:"–"; +} diff --git a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja index aa8f4031d..4cf41aef0 100644 --- a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja +++ b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja @@ -1,11 +1,11 @@ {%- if not document_tree_iterator.is_empty_tree() -%} -
    +
    {%- for folder_or_file in document_tree_iterator.iterator_files_first(): -%} {%- if folder_or_file.is_folder(): %} {%- if folder_or_file.files|length > 0 %}
    {% include "_res/svg__separator.jinja.html" %}{{ folder_or_file.rel_path }}
    @@ -17,11 +17,13 @@ {% set document_modified = not other_stats.contains_document_md5(document_md5) %}
    {% if document_modified %} - open + {# open #} {% endif %} - > + {% include "_res/svg_ico16_document.jinja.html" %} - .container { - display: flex; - } - .column { - flex: 1; /* This ensures each column takes equal width */ - padding: 10px; /* Optional, for some internal spacing */ - } - - main .main ul { - list-style-type: none; - } - - main .main summary {list-style: none} - main .main summary::-webkit-details-marker {display: none; } - main .main details > summary::before { - content:"+"; - padding-right: 0.5em; - } - main .main details[open] > summary::before { - content:"–"; - } - {% endblock head_css %} {% block head_scripts %} diff --git a/strictdoc/export/html/templates/screens/git/main.jinja b/strictdoc/export/html/templates/screens/git/main.jinja index 5eabe97ee..7eb174923 100644 --- a/strictdoc/export/html/templates/screens/git/main.jinja +++ b/strictdoc/export/html/templates/screens/git/main.jinja @@ -2,8 +2,9 @@ {% include "screens/git/form.jinja" %} {% if results %} -
    -
    + +
    +
    {% with document_tree=document_tree_lhs, document_tree_iterator=documents_iterator_lhs, @@ -14,8 +15,10 @@ %} {% include "screens/git/frame_project_tree.jinja" %} {% endwith %} +
    -
    +
    +
    {% with document_tree=document_tree_rhs, document_tree_iterator=documents_iterator_rhs, @@ -26,7 +29,7 @@ %} {% include "screens/git/frame_project_tree.jinja" %} {% endwith %} +
    -
    {% endif %} -
    \ No newline at end of file +
    diff --git a/strictdoc/export/html/templates/screens/git/node.jinja b/strictdoc/export/html/templates/screens/git/node.jinja index c2eafa32b..0327b77e3 100644 --- a/strictdoc/export/html/templates/screens/git/node.jinja +++ b/strictdoc/export/html/templates/screens/git/node.jinja @@ -1,6 +1,6 @@ -
      + {% if node.free_texts|length > 0 %} -
    • + {% set free_text = node.free_texts[0] %} {% set free_text_md5 = self_stats.get_md5_by_node(free_text) %} {% set free_text_modified = not other_stats.contains_free_text_md5(free_text_md5) %} @@ -28,11 +28,11 @@ {{ free_text.get_parts_as_text_escaped() }}
    -
    • + {% endif %} {%- for node_ in node.section_contents -%} -
  • + {%- if node_.is_section -%} {% set section = node_ %} {% set section_md5 = self_stats.get_md5_by_node(section) %} @@ -89,6 +89,6 @@ {% endwith %} {%- endif -%} -
    • + {% endfor %} - + From 9b6ccb56e0a466aca30ccec47c5af89a9ee045be Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 01:36:10 +0100 Subject: [PATCH 09/18] export/html: WIP: update diff CSS and markup; add text-icons --- strictdoc/export/html/_static/content.css | 83 ------------ strictdoc/export/html/_static/diff.css | 118 ++++++++++++++++++ .../screens/git/frame_project_tree.jinja | 11 +- .../html/templates/screens/git/index.jinja | 1 + .../html/templates/screens/git/node.jinja | 44 ++----- 5 files changed, 136 insertions(+), 121 deletions(-) create mode 100644 strictdoc/export/html/_static/diff.css diff --git a/strictdoc/export/html/_static/content.css b/strictdoc/export/html/_static/content.css index dc5cc247b..3f75ab8c9 100644 --- a/strictdoc/export/html/_static/content.css +++ b/strictdoc/export/html/_static/content.css @@ -171,86 +171,3 @@ sdoc-main-placeholder.page-tips { color: unset; opacity: 0.5; } - -/* diff */ - -.diff { - overflow: auto; - overflow-y: scroll; - overflow-wrap: break-word; - /* border: 1px solid var(--thumbBG); */ - border-radius: 4px; - border: 1px solid var(--color-border); -} - -.diff[left] { - direction:rtl; -} - -.diff[right] { - direction:ltr; -} - -.diff_inner { - direction: initial; -} - -.diff_content { - display: flex; - flex-direction: column; - gap: var(--base-rhythm); - padding: var(--base-rhythm); -} - -.diff_folder { - display: flex; - align-items: center; - justify-content: flex-start; - column-gap: calc(var(--base-rhythm)/2); - font-size: var(--font-size-sm); - min-width: 0; - line-height: 1.5; - padding-top: var(--base-rhythm); -} - -.diff_document { - background-color: white; - border-radius: 4px; - padding: var(--base-rhythm); -} - -.diff_document > summary { - line-height: 1.2; -} - -.diff_document > summary:hover { - color: var(--color-fg-accent); -} - -.diff details.diff_document > summary::before, -.diff details.diff_document[open] > summary::before { - content: none; -} - -.diff details { - width: 100%; -} - -.diff summary { - list-style: none; - display: flex; - gap: var(--base-rhythm); - cursor: pointer; -} - -.diff summary::-webkit-details-marker { - display: none; -} - -.diff details > summary::before { - content:"+"; -} - -.diff details[open] > summary::before { - content:"–"; -} diff --git a/strictdoc/export/html/_static/diff.css b/strictdoc/export/html/_static/diff.css new file mode 100644 index 000000000..260653716 --- /dev/null +++ b/strictdoc/export/html/_static/diff.css @@ -0,0 +1,118 @@ +/* diff */ + +/* columns */ +.diff { + overflow: auto; + overflow-y: scroll; + overflow-wrap: break-word; + /* border: 1px solid var(--thumbBG); */ + border-radius: 4px; + border: 1px solid var(--color-border); +} + +.diff[left] { + direction:rtl; +} + +.diff[right] { + direction:ltr; +} + +.diff_inner { + direction: initial; +} + +.diff_content { + display: flex; + flex-direction: column; + gap: var(--base-rhythm); + padding: var(--base-rhythm); +} + +/* details with summary */ + +.diff details { + width: 100%; + + padding: calc(var(--base-rhythm)/2); + padding-right: 0; + /* border: 1px solid transparent; */ + border-left: 4px solid transparent; +} + +.diff details[modified] { + border-color: var(--color-fg-accent); +} + +.diff summary { + list-style: none; + display: flex; + gap: var(--base-rhythm); + cursor: pointer; + color: var(--color-link); +} + +.diff details[modified] > summary { + color: var(--color-fg-accent); +} + +.diff summary:hover, +.diff details[modified] > summary:hover { + color: var(--color-hover); +} + +.diff summary::-webkit-details-marker { + display: none; +} + +.diff details > summary::before { + content:"+"; +} + +.diff details[open] > summary::before { + content:"–"; +} + +/* document / details */ + +details.diff_document { + background-color: white; + border-radius: 4px; + padding: 0 var(--base-rhythm); + border: 1px solid transparent; + border-left: 4px solid transparent; +} + +.diff_document[modified] { + border-color: var(--color-fg-accent); +} + +.diff_document > summary { + line-height: 1.2; + padding: var(--base-rhythm) 0; +} + +/* folder */ + +.diff_folder { + display: flex; + align-items: center; + justify-content: flex-start; + column-gap: calc(var(--base-rhythm)/2); + font-size: var(--font-size-sm); + min-width: 0; + line-height: 1.5; + padding-top: var(--base-rhythm); +} + +/* text_icon */ + +.text_icon::before { + content: attr(text); + padding: 0 calc(var(--base-rhythm)/2); + border: 1px solid; + border-radius: calc(var(--base-rhythm)/2); + font-size: var(--font-size-xxsm); + font-weight: 600; + text-transform: uppercase; +} diff --git a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja index 4cf41aef0..36e5e7f57 100644 --- a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja +++ b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja @@ -18,19 +18,16 @@
    {% if document_modified %} {# open #} {% endif %} {% include "_res/svg_ico16_document.jinja.html" %} - - {{ document_.title }} - + {{ document_.title }} {% with node=document_ %} {% include "screens/git/node.jinja" %} diff --git a/strictdoc/export/html/templates/screens/git/index.jinja b/strictdoc/export/html/templates/screens/git/index.jinja index 0244655d2..5f2ecd3ec 100644 --- a/strictdoc/export/html/templates/screens/git/index.jinja +++ b/strictdoc/export/html/templates/screens/git/index.jinja @@ -3,6 +3,7 @@ {% block head_css %} {{ super() }} + {% endblock head_css %} {% block head_scripts %} diff --git a/strictdoc/export/html/templates/screens/git/node.jinja b/strictdoc/export/html/templates/screens/git/node.jinja index 0327b77e3..0df91fdb9 100644 --- a/strictdoc/export/html/templates/screens/git/node.jinja +++ b/strictdoc/export/html/templates/screens/git/node.jinja @@ -7,24 +7,14 @@
    - - ... - + - + {{ free_text.get_parts_as_text_escaped() }}
    @@ -40,19 +30,15 @@
    - - + + {{ section.context.title_number_string if section.context.title_number_string else " " * (section.ng_level * 2 - 1) }} - {{ section.title }} - + + {{ section.title }} {% with node=section %} {% include "screens/git/node.jinja" %} @@ -66,23 +52,19 @@
    - - + + {{ section.context.title_number_string if section.context.title_number_string else " " * (section.ng_level * 2 - 1) }} {%- if section.reserved_title is not none -%} + {{ section.reserved_title }} - + {%- endif -%} - {% with requirement=section, requirement_modified=requirement_modified %} {% include "screens/git/requirement.jinja" %} From 11becb59d358cb3b6584a2b93ce713db0f3c2294 Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Sun, 3 Dec 2023 01:36:29 +0100 Subject: [PATCH 10/18] UI: Diff Screen: better color coding of free text changes --- .../html/templates/screens/git/node.jinja | 17 ++- strictdoc/git/project_diff_analyzer.py | 112 +++++++++++++++++- strictdoc/helpers/diff.py | 11 +- 3 files changed, 133 insertions(+), 7 deletions(-) diff --git a/strictdoc/export/html/templates/screens/git/node.jinja b/strictdoc/export/html/templates/screens/git/node.jinja index 0df91fdb9..363e694a3 100644 --- a/strictdoc/export/html/templates/screens/git/node.jinja +++ b/strictdoc/export/html/templates/screens/git/node.jinja @@ -14,8 +14,21 @@
    - - {{ free_text.get_parts_as_text_escaped() }} + + {% set colored_diff = other_stats.get_diffed_free_text(node, side) %} + {% if colored_diff is not none %} + {{ colored_diff }} + {% else %} + {{ free_text.get_parts_as_text_escaped() }} + {% endif %} diff --git a/strictdoc/git/project_diff_analyzer.py b/strictdoc/git/project_diff_analyzer.py index f21295a07..33fd37933 100644 --- a/strictdoc/git/project_diff_analyzer.py +++ b/strictdoc/git/project_diff_analyzer.py @@ -1,7 +1,7 @@ import hashlib import statistics from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, Union from strictdoc.backend.sdoc.models.document import Document from strictdoc.backend.sdoc.models.reference import ( @@ -59,6 +59,7 @@ class ProjectTreeDiffStats: map_uid_to_nodes: Dict[str, Any] = field(default_factory=dict) map_titles_to_nodes: Dict[str, List] = field(default_factory=dict) map_statements_to_nodes: Dict[str, Any] = field(default_factory=dict) + map_rel_paths_to_docs: Dict[str, Document] = field(default_factory=dict) cache_requirement_to_requirement: Dict[Requirement, Requirement] = field( default_factory=dict @@ -102,6 +103,107 @@ def contains_requirement_field( return True return False + def get_diffed_free_text(self, node: Union[Section, Document], side: str): + assert isinstance(node, (Section, Document)) + assert side in ("left", "right") + + if isinstance(node, Document): + document: Document = assert_cast(node, Document) + + other_document_or_none: Optional[ + Document + ] = self.map_rel_paths_to_docs.get( + document.meta.input_doc_full_path + ) + if other_document_or_none is None: + return None + other_document: Document = assert_cast( + other_document_or_none, Document + ) + if len(other_document.free_texts) == 0: + return None + document_free_text = document.free_texts[0] + other_document_free_text = other_document.free_texts[0] + document_free_text_parts = ( + document_free_text.get_parts_as_text_escaped() + ) + other_document_free_text_parts = ( + other_document_free_text.get_parts_as_text_escaped() + ) + if side == "left": + return get_colored_diff_string( + document_free_text_parts, + other_document_free_text_parts, + side, + ) + else: + return get_colored_diff_string( + other_document_free_text_parts, + document_free_text_parts, + side, + ) + + if isinstance(node, Section): + section: Section = assert_cast(node, Section) + section_free_text = section.free_texts[0] + section_free_text_parts = ( + section_free_text.get_parts_as_text_escaped() + ) + + if section.reserved_uid is not None: + other_section_or_none: Optional[ + Section + ] = self.map_uid_to_nodes.get(section.reserved_uid) + if other_section_or_none is not None: + other_section: Section = assert_cast( + other_section_or_none, Section + ) + + if len(other_section.free_texts) > 0: + other_section_free_text = other_section.free_texts[0] + other_section_free_text_parts = ( + other_section_free_text.get_parts_as_text_escaped() + ) + + if side == "left": + return get_colored_diff_string( + section_free_text_parts, + other_section_free_text_parts, + side, + ) + else: + return get_colored_diff_string( + other_section_free_text_parts, + section_free_text_parts, + side, + ) + else: + if side == "left": + return get_colored_diff_string( + section_free_text_parts, "", side + ) + else: + return get_colored_diff_string( + "", section_free_text_parts, side + ) + else: + if side == "left": + return get_colored_diff_string( + section_free_text_parts, "", side + ) + else: + return get_colored_diff_string( + "", section_free_text_parts, side + ) + + # Section does not have a UID. We can still try to find a section + # with the same title if it still exists in the same parent + # section/document scope. + else: + pass + + return None + def get_diffed_requirement_field( self, requirement: Requirement, @@ -242,6 +344,10 @@ def analyze_document( map_nodes_to_hashers: Dict[Any, Any] = {document: hashlib.md5()} + document_tree_stats.map_rel_paths_to_docs[ + document.meta.input_doc_full_path + ] = document + # Document's top level free text. if len(document.free_texts) > 0: free_text = document.free_texts[0] @@ -252,6 +358,10 @@ def analyze_document( for node in document_iterator.all_content(): if isinstance(node, Section): + if node.reserved_uid is not None: + document_tree_stats.map_uid_to_nodes[ + node.reserved_uid + ] = node hasher = hashlib.md5() hasher.update(node.title.encode("utf-8")) if len(node.free_texts) > 0: diff --git a/strictdoc/helpers/diff.py b/strictdoc/helpers/diff.py index 225f2eff2..f02a1e68e 100644 --- a/strictdoc/helpers/diff.py +++ b/strictdoc/helpers/diff.py @@ -6,10 +6,13 @@ def similar(a, b): return SequenceMatcher(None, a, b).ratio() -red = lambda text: f'{text}' -green = lambda text: f'{text}' -blue = lambda text: f'{text}' -white = lambda text: f'{text}' +red = ( + lambda text: f'{text}' +) +green = ( + lambda text: f'{text}' +) +white = lambda text: f"{text}" def get_colored_diff_string(old, new, flag: str): From 98567a6c19367fc00d92d6f7a42b25c1add7b690 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 01:59:43 +0100 Subject: [PATCH 11/18] export/html: WIP: Add a border below the summary when the details is open --- strictdoc/export/html/_static/diff.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/strictdoc/export/html/_static/diff.css b/strictdoc/export/html/_static/diff.css index 260653716..e269ba646 100644 --- a/strictdoc/export/html/_static/diff.css +++ b/strictdoc/export/html/_static/diff.css @@ -92,6 +92,10 @@ details.diff_document { padding: var(--base-rhythm) 0; } +.diff_document[modified][open] > summary { + border-bottom: 1px dotted; +} + /* folder */ .diff_folder { From 8659f4aa0d298ee0a86a359097f9ef4d05db4715 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 07:26:20 +0100 Subject: [PATCH 12/18] export/html: DIFF: update CSS and markup --- strictdoc/export/html/_static/diff.css | 164 ++++++++++++++++-- .../html/templates/screens/git/fields.jinja | 44 +++++ .../screens/git/frame_project_tree.jinja | 25 ++- .../templates/screens/git/free_text.jinja | 27 +++ .../html/templates/screens/git/node.jinja | 89 ---------- .../templates/screens/git/requirement.jinja | 69 +++----- .../html/templates/screens/git/section.jinja | 22 +++ strictdoc/git/project_diff_analyzer.py | 15 +- strictdoc/helpers/diff.py | 4 +- 9 files changed, 298 insertions(+), 161 deletions(-) create mode 100644 strictdoc/export/html/templates/screens/git/fields.jinja create mode 100644 strictdoc/export/html/templates/screens/git/free_text.jinja delete mode 100644 strictdoc/export/html/templates/screens/git/node.jinja create mode 100644 strictdoc/export/html/templates/screens/git/section.jinja diff --git a/strictdoc/export/html/_static/diff.css b/strictdoc/export/html/_static/diff.css index e269ba646..0672fa7df 100644 --- a/strictdoc/export/html/_static/diff.css +++ b/strictdoc/export/html/_static/diff.css @@ -1,4 +1,19 @@ /* diff */ +:root { + --pre-stripe: 20px; + --pre-stripe-color: rgba(0,0,0,0.02); + --pre-block-bg-color: rgba(0,0,0,0.01); + + --diff-block-color-left: rgba(255, 55, 55, .05); + --diff-word-color-left: rgba(255, 55, 55, .2); + --diff-icon-color-left: rgba(255, 55, 55, .75); + --diff-document-color-left: rgba(255, 55, 55, 1); + + --diff-block-color-right: rgba(20, 120, 20, .05); + --diff-word-color-right: rgba(20, 120, 20, .2); + --diff-icon-color-right: rgba(20, 120, 20, .75); + --diff-document-color-right: rgba(20, 120, 20, 1); +} /* columns */ .diff { @@ -8,6 +23,9 @@ /* border: 1px solid var(--thumbBG); */ border-radius: 4px; border: 1px solid var(--color-border); + + /* for position:sticky*/ + position: relative; } .diff[left] { @@ -33,15 +51,6 @@ .diff details { width: 100%; - - padding: calc(var(--base-rhythm)/2); - padding-right: 0; - /* border: 1px solid transparent; */ - border-left: 4px solid transparent; -} - -.diff details[modified] { - border-color: var(--color-fg-accent); } .diff summary { @@ -56,6 +65,14 @@ color: var(--color-fg-accent); } +.diff details[modified="left"] > summary { + color: var(--diff-document-color-left); +} + +.diff details[modified="right"] > summary { + color: var(--diff-document-color-right); +} + .diff summary:hover, .diff details[modified] > summary:hover { color: var(--color-hover); @@ -76,26 +93,63 @@ /* document / details */ details.diff_document { - background-color: white; + background-color: var(--color-bg-contrast); + border: 1px solid transparent; border-radius: 4px; padding: 0 var(--base-rhythm); - border: 1px solid transparent; - border-left: 4px solid transparent; } -.diff_document[modified] { +details.diff_document[modified] { border-color: var(--color-fg-accent); } +details.diff_document[modified="left"] { + border-color: var(--diff-document-color-left, rgba(255, 55, 55, 1)); +} + +details.diff_document[modified="right"] { + border-color: var(--diff-document-color-right, rgba(20, 120, 20, 1)); +} + .diff_document > summary { line-height: 1.2; padding: var(--base-rhythm) 0; + justify-content: space-between; + align-items: center; +} + +.diff_document[open] > summary { + /* + When the details are closed up, the summary sometimes covers the bottom border. + That's why we only add the background for cases where the card is open + and sticking makes sense. + */ + background: var(--color-bg-contrast); + position: sticky; + top: 0; +} + +.diff_document > summary .document_title { + flex-grow: 1; } -.diff_document[modified][open] > summary { +.diff_document[open] > summary { border-bottom: 1px dotted; } +.diff_document > summary::before, +.diff_document[open] > summary::before { + content: none !important; +} + +.diff_document > summary::after { + content:"+"; +} + +.diff_document[open] > summary::after { + content:"–"; +} + /* folder */ .diff_folder { @@ -109,6 +163,40 @@ details.diff_document { padding-top: var(--base-rhythm); } +/* node content */ + +.diff_node { + margin: var(--base-rhythm) 0; +} + +.diff_node_fields { + display: flex; + flex-direction: column; + row-gap: var(--base-rhythm); + + /* in node and in doc as Abstract: */ + padding-top: var(--base-rhythm); +} + +.diff_node > .diff_node_fields { + /* in node only: */ + margin-bottom: calc(var(--base-rhythm)*2); + padding-left: calc(var(--base-rhythm)*2); + + /* padding-left: calc(var(--base-rhythm)*1.5); + border-left: 1px dotted; */ + + /* border-left: var(--base-rhythm) solid rgba(0,0,0,0.01); + padding-left: calc(var(--base-rhythm)*1); */ +} + +.diff_node_field { + display: flex; + flex-wrap: wrap; + align-items: baseline; + column-gap: var(--base-rhythm); +} + /* text_icon */ .text_icon::before { @@ -120,3 +208,51 @@ details.diff_document { font-weight: 600; text-transform: uppercase; } + +/* in summary: */ +[modified="left"] > summary .text_icon::before, +/* in field: */ +[modified="left"] > .text_icon::before { + color: white; + background-color: var(--diff-icon-color-left); + border-color: var(--diff-icon-color-left); +} + +/* in summary: */ +[modified="right"] > summary .text_icon::before, +/* in field: */ +[modified="right"] > .text_icon::before { + color: white; + background-color: var(--diff-icon-color-right); + border-color: var(--diff-icon-color-right); +} + +/* pre */ + +.sdoc_pre_content { + flex-grow: 1; + white-space: pre-wrap; + font-family: monospace; + font-size: 0.85em; + line-height: var(--pre-stripe); + background-image: repeating-linear-gradient( + to bottom, + var(--pre-stripe-color), + var(--pre-stripe-color) var(--pre-stripe), + transparent var(--pre-stripe), + transparent calc(var(--pre-stripe)*2) + ); + background-color: var(--pre-block-bg-color); +} + +[modified="left"] > .sdoc_pre_content { + background-color: var(--diff-block-color-left); +} +.lambda_red {background-color: var(--diff-word-color-left);} + +[modified="right"] > .sdoc_pre_content { + background-color: var(--diff-block-color-right); +} +.lambda_green {background-color: var(--diff-word-color-right);} + + diff --git a/strictdoc/export/html/templates/screens/git/fields.jinja b/strictdoc/export/html/templates/screens/git/fields.jinja new file mode 100644 index 000000000..f579d0921 --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/fields.jinja @@ -0,0 +1,44 @@ +
    + {%- for meta_field in requirement.enumerate_all_fields() -%} + {%- set field_modified = requirement_modified and not other_stats.contains_requirement_field(requirement, meta_field[0], meta_field[1]) -%} +
    + + + {%- if field_modified -%} + {{ other_stats.get_diffed_requirement_field(requirement, meta_field[0], meta_field[1], side) }} + {%- else -%} + {{ meta_field[1] }} + {%- endif -%} + +
    + {%- endfor -%} + + {%- if traceability_index.has_parent_requirements(requirement) %} + {%- for parent_requirement_, relation_role_ in traceability_index.get_parent_relations_with_roles(requirement) %} + {# Relations are rendered one field at a time with its own label, not a group of fields: #} +
    + +
    + {%- if true -%} +{{ parent_requirement_.reserved_uid }}{# Warning, we're inside a PRE and the line break here is significant: #} +{{ parent_requirement_.reserved_title if parent_requirement_.reserved_title else "" }} + {%- endif -%} + {%- if relation_role_ is not none -%} + ({{ relation_role_ }}) + {%- endif -%} +
    +
    + {%- endfor -%} + {%- endif -%} + +
    {# //.diff_node_fields #} diff --git a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja index 36e5e7f57..f01f912e0 100644 --- a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja +++ b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja @@ -11,15 +11,15 @@ >{% include "_res/svg__separator.jinja.html" %}{{ folder_or_file.rel_path }} {% endif %} {% else %} - {%- set document_ = document_tree.get_document_by_path(folder_or_file.get_full_path()) %} + {%- set document = document_tree.get_document_by_path(folder_or_file.get_full_path()) %} - {% set document_md5 = self_stats.get_md5_by_node(document_) %} + {% set document_md5 = self_stats.get_md5_by_node(document) %} {% set document_modified = not other_stats.contains_document_md5(document_md5) %}
    {% if document_modified %} @@ -27,11 +27,24 @@ {% endif %} {% include "_res/svg_ico16_document.jinja.html" %} - {{ document_.title }} + {{ document.title }} - {% with node=document_ %} - {% include "screens/git/node.jinja" %} + + {% with text_icon="abstract", node=document %} + {% include "screens/git/free_text.jinja" %} {% endwith %} + + {%- set document_iterator = traceability_index.get_document_iterator(document) -%} + {%- for section_or_requirement in document_iterator.all_content() %} + {%- if section_or_requirement.is_requirement %} + {%- set requirement = section_or_requirement %} + {% include "screens/git/requirement.jinja" %} + + {%- elif section_or_requirement.is_section %} + {%- set section = section_or_requirement %} + {% include "screens/git/section.jinja" %} + {%- endif %} + {%- endfor -%}
    {% endif %} diff --git a/strictdoc/export/html/templates/screens/git/free_text.jinja b/strictdoc/export/html/templates/screens/git/free_text.jinja new file mode 100644 index 000000000..36e9913ef --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/free_text.jinja @@ -0,0 +1,27 @@ +{% if node.free_texts|length > 0 %} +{% set free_text = node.free_texts[0] %} +{% set free_text_md5 = self_stats.get_md5_by_node(free_text) %} +{% set free_text_modified = not other_stats.contains_free_text_md5(free_text_md5) %} +
    +
    + + + + +
    + {%- set colored_diff = other_stats.get_diffed_free_text(node, side) -%} + {%- if colored_diff is not none -%} + {{ colored_diff }} + {%- else -%} + {{ free_text.get_parts_as_text_escaped() }} + {%- endif -%} +
    +
    +
    +{% endif %} diff --git a/strictdoc/export/html/templates/screens/git/node.jinja b/strictdoc/export/html/templates/screens/git/node.jinja deleted file mode 100644 index 363e694a3..000000000 --- a/strictdoc/export/html/templates/screens/git/node.jinja +++ /dev/null @@ -1,89 +0,0 @@ - - {% if node.free_texts|length > 0 %} - - {% set free_text = node.free_texts[0] %} - {% set free_text_md5 = self_stats.get_md5_by_node(free_text) %} - {% set free_text_modified = not other_stats.contains_free_text_md5(free_text_md5) %} - -
    - - - - - - {% set colored_diff = other_stats.get_diffed_free_text(node, side) %} - {% if colored_diff is not none %} - {{ colored_diff }} - {% else %} - {{ free_text.get_parts_as_text_escaped() }} - {% endif %} - -
    - - {% endif %} - - {%- for node_ in node.section_contents -%} - - {%- if node_.is_section -%} - {% set section = node_ %} - {% set section_md5 = self_stats.get_md5_by_node(section) %} - {% set section_modified = not other_stats.contains_section_md5(section_md5) %} - -
    - - - - {{ section.context.title_number_string if section.context.title_number_string else " " * (section.ng_level * 2 - 1) }} - - {{ section.title }} - - {% with node=section %} - {% include "screens/git/node.jinja" %} - {% endwith %} -
    - - {%- else -%}{# if requirement #} - {% set section = node_ %} - {% set requirement_md5 = self_stats.get_md5_by_node(section) %} - {% set requirement_modified = not other_stats.contains_requirement_md5(requirement_md5) %} - -
    - - - - {{ section.context.title_number_string if section.context.title_number_string else " " * (section.ng_level * 2 - 1) }} - - {%- if section.reserved_title is not none -%} - - {{ section.reserved_title }} - - {%- endif -%} - - {% with requirement=section, requirement_modified=requirement_modified %} - {% include "screens/git/requirement.jinja" %} - {% endwith %} -
    - {%- endif -%} - - {% endfor %} - diff --git a/strictdoc/export/html/templates/screens/git/requirement.jinja b/strictdoc/export/html/templates/screens/git/requirement.jinja index d5d7f1d16..64a91af08 100644 --- a/strictdoc/export/html/templates/screens/git/requirement.jinja +++ b/strictdoc/export/html/templates/screens/git/requirement.jinja @@ -1,48 +1,25 @@ -
    - {% for meta_field in requirement.enumerate_all_fields() %} - {% set field_modified = requirement_modified and not other_stats.contains_requirement_field(requirement, meta_field[0], meta_field[1]) %} -
    - {% if field_modified %} - {{ other_stats.get_diffed_requirement_field(requirement, meta_field[0], meta_field[1], side) }} - {% else %} - {{ meta_field[0] }}: {{ meta_field[1] }} - {% endif %} -
    - {% endfor %} +{% set requirement_md5 = self_stats.get_md5_by_node(requirement) %} +{% set requirement_modified = not other_stats.contains_requirement_md5(requirement_md5) %} - {%- if traceability_index.has_parent_requirements(requirement) %} -
    Parent relations:
    -
    - -
    - {%- endif %} +
    + + + + {{ requirement.context.title_number_string if requirement.context.title_number_string else " " * (requirement.ng_level * 2 - 1) }} + + {%- if requirement.reserved_title is not none -%} + + {{ requirement.reserved_title }} + + {%- endif -%} + -
    + {% include "screens/git/fields.jinja" %} + + diff --git a/strictdoc/export/html/templates/screens/git/section.jinja b/strictdoc/export/html/templates/screens/git/section.jinja new file mode 100644 index 000000000..d73d5ac48 --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/section.jinja @@ -0,0 +1,22 @@ +{% set section_md5 = self_stats.get_md5_by_node(section) %} +{% set section_modified = not other_stats.contains_section_md5(section_md5) %} + +
    + + + + {{ section.context.title_number_string if section.context.title_number_string else " " * (section.ng_level * 2 - 1) }} + + {{ section.title }} + + + {% with text_icon="free text", node=section %} + {% include "screens/git/free_text.jinja" %} + {% endwith %} + +
    diff --git a/strictdoc/git/project_diff_analyzer.py b/strictdoc/git/project_diff_analyzer.py index 33fd37933..d06225dc8 100644 --- a/strictdoc/git/project_diff_analyzer.py +++ b/strictdoc/git/project_diff_analyzer.py @@ -221,7 +221,7 @@ def get_diffed_requirement_field( other_requirement is None or field_name not in other_requirement.ordered_fields_lookup ): - return field_name + ": " + field_value + return field_value other_requirement_fields = other_requirement.ordered_fields_lookup[ field_name @@ -241,12 +241,12 @@ def get_diffed_requirement_field( colored_field_value = get_colored_diff_string( field_value, other_field_value, side ) - return field_name + ": " + colored_field_value + return colored_field_value else: colored_field_value = get_colored_diff_string( other_field_value, field_value, side ) - return field_name + ": " + colored_field_value + return colored_field_value def contains_requirement_relations( self, @@ -437,7 +437,14 @@ def recurse(node): map_nodes_to_hashers[node].update(node_md5) return map_nodes_to_hashers[node].hexdigest().encode("utf-8") - recurse(document) + # recurse(document) + for node_ in document_iterator.all_content(): + node_md5 = ( + map_nodes_to_hashers[node_] + .hexdigest() + .encode("utf-8") + ) + map_nodes_to_hashers[document].update(node_md5) for node_, node_hasher_ in map_nodes_to_hashers.items(): node_md5 = node_hasher_.hexdigest() diff --git a/strictdoc/helpers/diff.py b/strictdoc/helpers/diff.py index f02a1e68e..6dab89958 100644 --- a/strictdoc/helpers/diff.py +++ b/strictdoc/helpers/diff.py @@ -7,10 +7,10 @@ def similar(a, b): red = ( - lambda text: f'{text}' + lambda text: f'{text}' ) green = ( - lambda text: f'{text}' + lambda text: f'{text}' ) white = lambda text: f"{text}" From 0a6c3aafc53771a0b4a114f621f54e686f6a5df3 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 09:04:00 +0100 Subject: [PATCH 13/18] export/html: DIFF: add JS controller for bulk opening and closing --- strictdoc/export/html/_static/content.css | 2 +- .../_static/controllers/diff_controller.js | 47 +++++++++++++++++++ strictdoc/export/html/_static/diff.css | 22 +++++++++ .../html/templates/screens/git/index.jinja | 1 + .../html/templates/screens/git/main.jinja | 10 +++- 5 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 strictdoc/export/html/_static/controllers/diff_controller.js diff --git a/strictdoc/export/html/_static/content.css b/strictdoc/export/html/_static/content.css index 3f75ab8c9..bb96ddf05 100644 --- a/strictdoc/export/html/_static/content.css +++ b/strictdoc/export/html/_static/content.css @@ -26,7 +26,7 @@ place-content: stretch stretch; grid-template-columns: minmax(0, 1fr) /* issue#1370 https://css-tricks.com/preventing-a-grid-blowout/ */ minmax(0, 1fr); - grid-template-rows: minmax(0, max-content) minmax(0, 1fr); + grid-template-rows: minmax(0, max-content) minmax(0, max-content); gap: var(--base-rhythm); } diff --git a/strictdoc/export/html/_static/controllers/diff_controller.js b/strictdoc/export/html/_static/controllers/diff_controller.js new file mode 100644 index 000000000..5d8f93a5d --- /dev/null +++ b/strictdoc/export/html/_static/controllers/diff_controller.js @@ -0,0 +1,47 @@ +Stimulus.register("diff", class extends Controller { + initialize() { + + const leftColumn = this.element.querySelector('.diff[left]'); + const rightColumn = this.element.querySelector('.diff[right]'); + + const leftOpenBtn = document.getElementById('diff_left_open'); + const leftCloseBtn = document.getElementById('diff_left_close'); + const rightOpenBtn = document.getElementById('diff_right_open'); + const rightCloseBtn = document.getElementById('diff_right_close'); + + const detailsLeftAll = [...leftColumn.querySelectorAll('details')]; + const detailsRightAll = [...rightColumn.querySelectorAll('details')]; + const detailsLeft = [...leftColumn.querySelectorAll('details[modified]')]; + const detailsRight = [...rightColumn.querySelectorAll('details[modified]')]; + + // Add event listener + leftOpenBtn.addEventListener("click", function(event){ + event.preventDefault(); + openAll(detailsLeft); + }); + leftCloseBtn.addEventListener("click", function(event){ + event.preventDefault(); + closeAll(detailsLeftAll); + }); + rightOpenBtn.addEventListener("click", function(event){ + event.preventDefault(); + openAll(detailsRight); + }); + rightCloseBtn.addEventListener("click", function(event){ + event.preventDefault(); + closeAll(detailsRightAll); + }); + + } +}); + +function closeAll(details) { + details.forEach(detail => { + detail.removeAttribute("open") + }); +} +function openAll(details) { + details.forEach(detail => { + detail.setAttribute("open", "") + }); +} diff --git a/strictdoc/export/html/_static/diff.css b/strictdoc/export/html/_static/diff.css index 0672fa7df..c3e5a8340 100644 --- a/strictdoc/export/html/_static/diff.css +++ b/strictdoc/export/html/_static/diff.css @@ -15,6 +15,28 @@ --diff-document-color-right: rgba(20, 120, 20, 1); } +.diff_controls { + text-align: center; +} + +#diff_left_open { + color: var(--diff-document-color-left); +} + +#diff_right_open { + color: var(--diff-document-color-right); +} + +#diff_left_close, +#diff_right_close { + color: var(--color-link); +} + +#diff_left_open:hover, +#diff_right_open:hover { + color: var(--color-hover); +} + /* columns */ .diff { overflow: auto; diff --git a/strictdoc/export/html/templates/screens/git/index.jinja b/strictdoc/export/html/templates/screens/git/index.jinja index 5f2ecd3ec..dc1b38f48 100644 --- a/strictdoc/export/html/templates/screens/git/index.jinja +++ b/strictdoc/export/html/templates/screens/git/index.jinja @@ -17,6 +17,7 @@ window.Controller = Controller; + {%- endif -%} {% endblock head_scripts %} diff --git a/strictdoc/export/html/templates/screens/git/main.jinja b/strictdoc/export/html/templates/screens/git/main.jinja index 7eb174923..78a41776e 100644 --- a/strictdoc/export/html/templates/screens/git/main.jinja +++ b/strictdoc/export/html/templates/screens/git/main.jinja @@ -1,5 +1,13 @@ -
    +
    {% include "screens/git/form.jinja" %} +
    + +open all modified + –close all +
    +
    + +open all modified + –close all +
    {% if results %} From bc1157be506ab1e8db587b8a87b9b733d0539720 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 09:09:55 +0100 Subject: [PATCH 14/18] export/html: DIFF: fix color lines overflow in PRE block --- strictdoc/export/html/_static/diff.css | 1 + 1 file changed, 1 insertion(+) diff --git a/strictdoc/export/html/_static/diff.css b/strictdoc/export/html/_static/diff.css index c3e5a8340..8c6c42c6c 100644 --- a/strictdoc/export/html/_static/diff.css +++ b/strictdoc/export/html/_static/diff.css @@ -254,6 +254,7 @@ details.diff_document[modified="right"] { .sdoc_pre_content { flex-grow: 1; white-space: pre-wrap; + overflow-x: clip; font-family: monospace; font-size: 0.85em; line-height: var(--pre-stripe); From 795a860cca5ea48239bde86159801d7e00c0484c Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 22:30:26 +0100 Subject: [PATCH 15/18] export/html: DIFF: Remove collapsibility for section free text and leave it for the abstract. --- .../screens/git/document_abstract.jinja | 30 +++++++++++++++++++ .../screens/git/frame_project_tree.jinja | 4 +-- .../templates/screens/git/free_text.jinja | 30 ++++++++----------- 3 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 strictdoc/export/html/templates/screens/git/document_abstract.jinja diff --git a/strictdoc/export/html/templates/screens/git/document_abstract.jinja b/strictdoc/export/html/templates/screens/git/document_abstract.jinja new file mode 100644 index 000000000..f0e75445d --- /dev/null +++ b/strictdoc/export/html/templates/screens/git/document_abstract.jinja @@ -0,0 +1,30 @@ +{% set text_icon = "abstract" %} +{% set node = document %} + +{% if node.free_texts|length > 0 %} +{% set free_text = node.free_texts[0] %} +{% set free_text_md5 = self_stats.get_md5_by_node(free_text) %} +{% set free_text_modified = not other_stats.contains_free_text_md5(free_text_md5) %} +
    +
    + + + + +
    + {%- set colored_diff = other_stats.get_diffed_free_text(node, side) -%} + {%- if colored_diff is not none -%} + {{ colored_diff }} + {%- else -%} + {{ free_text.get_parts_as_text_escaped() }} + {%- endif -%} +
    +
    +
    +{% endif %} diff --git a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja index f01f912e0..eac87ac60 100644 --- a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja +++ b/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja @@ -30,9 +30,7 @@ {{ document.title }}
    - {% with text_icon="abstract", node=document %} - {% include "screens/git/free_text.jinja" %} - {% endwith %} + {% include "screens/git/document_abstract.jinja" %} {%- set document_iterator = traceability_index.get_document_iterator(document) -%} {%- for section_or_requirement in document_iterator.all_content() %} diff --git a/strictdoc/export/html/templates/screens/git/free_text.jinja b/strictdoc/export/html/templates/screens/git/free_text.jinja index 36e9913ef..0ddc372ca 100644 --- a/strictdoc/export/html/templates/screens/git/free_text.jinja +++ b/strictdoc/export/html/templates/screens/git/free_text.jinja @@ -3,25 +3,21 @@ {% set free_text_md5 = self_stats.get_md5_by_node(free_text) %} {% set free_text_modified = not other_stats.contains_free_text_md5(free_text_md5) %}
    -
    - - - -
    - {%- set colored_diff = other_stats.get_diffed_free_text(node, side) -%} - {%- if colored_diff is not none -%} - {{ colored_diff }} - {%- else -%} - {{ free_text.get_parts_as_text_escaped() }} - {%- endif -%} + +
    + {%- set colored_diff = other_stats.get_diffed_free_text(node, side) -%} + {%- if colored_diff is not none -%} + {{ colored_diff }} + {%- else -%} + {{ free_text.get_parts_as_text_escaped() }} + {%- endif -%} +
    -
    {% endif %} From 3420c47da3cc3b24161ed01b6d55558e31c1b0e7 Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 22:38:18 +0100 Subject: [PATCH 16/18] export/html: DIFF: file renaming --- .../git/{frame_project_tree.jinja => content.jinja} | 8 ++++---- .../screens/git/{ => fields}/document_abstract.jinja | 0 .../html/templates/screens/git/{ => fields}/fields.jinja | 0 .../templates/screens/git/{ => fields}/free_text.jinja | 0 strictdoc/export/html/templates/screens/git/main.jinja | 4 ++-- .../templates/screens/git/{ => node}/requirement.jinja | 2 +- .../html/templates/screens/git/{ => node}/section.jinja | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename strictdoc/export/html/templates/screens/git/{frame_project_tree.jinja => content.jinja} (87%) rename strictdoc/export/html/templates/screens/git/{ => fields}/document_abstract.jinja (100%) rename strictdoc/export/html/templates/screens/git/{ => fields}/fields.jinja (100%) rename strictdoc/export/html/templates/screens/git/{ => fields}/free_text.jinja (100%) rename strictdoc/export/html/templates/screens/git/{ => node}/requirement.jinja (92%) rename strictdoc/export/html/templates/screens/git/{ => node}/section.jinja (91%) diff --git a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja b/strictdoc/export/html/templates/screens/git/content.jinja similarity index 87% rename from strictdoc/export/html/templates/screens/git/frame_project_tree.jinja rename to strictdoc/export/html/templates/screens/git/content.jinja index eac87ac60..82597fa37 100644 --- a/strictdoc/export/html/templates/screens/git/frame_project_tree.jinja +++ b/strictdoc/export/html/templates/screens/git/content.jinja @@ -30,17 +30,17 @@ {{ document.title }}     - {% include "screens/git/document_abstract.jinja" %} + {% include "screens/git/fields/document_abstract.jinja" %} {%- set document_iterator = traceability_index.get_document_iterator(document) -%} {%- for section_or_requirement in document_iterator.all_content() %} {%- if section_or_requirement.is_requirement %} {%- set requirement = section_or_requirement %} - {% include "screens/git/requirement.jinja" %} + {% include "screens/git/node/requirement.jinja" %} {%- elif section_or_requirement.is_section %} {%- set section = section_or_requirement %} - {% include "screens/git/section.jinja" %} + {% include "screens/git/node/section.jinja" %} {%- endif %} {%- endfor -%} @@ -49,6 +49,6 @@ {%- endfor -%} {%- else -%} - 🐛 The project has no documents yet. + 🌚 The project has no documents yet. {%- endif -%} diff --git a/strictdoc/export/html/templates/screens/git/document_abstract.jinja b/strictdoc/export/html/templates/screens/git/fields/document_abstract.jinja similarity index 100% rename from strictdoc/export/html/templates/screens/git/document_abstract.jinja rename to strictdoc/export/html/templates/screens/git/fields/document_abstract.jinja diff --git a/strictdoc/export/html/templates/screens/git/fields.jinja b/strictdoc/export/html/templates/screens/git/fields/fields.jinja similarity index 100% rename from strictdoc/export/html/templates/screens/git/fields.jinja rename to strictdoc/export/html/templates/screens/git/fields/fields.jinja diff --git a/strictdoc/export/html/templates/screens/git/free_text.jinja b/strictdoc/export/html/templates/screens/git/fields/free_text.jinja similarity index 100% rename from strictdoc/export/html/templates/screens/git/free_text.jinja rename to strictdoc/export/html/templates/screens/git/fields/free_text.jinja diff --git a/strictdoc/export/html/templates/screens/git/main.jinja b/strictdoc/export/html/templates/screens/git/main.jinja index 78a41776e..82ac94617 100644 --- a/strictdoc/export/html/templates/screens/git/main.jinja +++ b/strictdoc/export/html/templates/screens/git/main.jinja @@ -21,7 +21,7 @@ other_stats=rhs_stats, side="left" %} - {% include "screens/git/frame_project_tree.jinja" %} + {% include "screens/git/content.jinja" %} {% endwith %} @@ -35,7 +35,7 @@ other_stats=lhs_stats, side="right" %} - {% include "screens/git/frame_project_tree.jinja" %} + {% include "screens/git/content.jinja" %} {% endwith %} diff --git a/strictdoc/export/html/templates/screens/git/requirement.jinja b/strictdoc/export/html/templates/screens/git/node/requirement.jinja similarity index 92% rename from strictdoc/export/html/templates/screens/git/requirement.jinja rename to strictdoc/export/html/templates/screens/git/node/requirement.jinja index 64a91af08..a40b6c095 100644 --- a/strictdoc/export/html/templates/screens/git/requirement.jinja +++ b/strictdoc/export/html/templates/screens/git/node/requirement.jinja @@ -20,6 +20,6 @@ {%- endif -%} - {% include "screens/git/fields.jinja" %} + {% include "screens/git/fields/fields.jinja" %} diff --git a/strictdoc/export/html/templates/screens/git/section.jinja b/strictdoc/export/html/templates/screens/git/node/section.jinja similarity index 91% rename from strictdoc/export/html/templates/screens/git/section.jinja rename to strictdoc/export/html/templates/screens/git/node/section.jinja index d73d5ac48..4a3bd65b9 100644 --- a/strictdoc/export/html/templates/screens/git/section.jinja +++ b/strictdoc/export/html/templates/screens/git/node/section.jinja @@ -16,7 +16,7 @@ {% with text_icon="free text", node=section %} - {% include "screens/git/free_text.jinja" %} + {% include "screens/git/fields/free_text.jinja" %} {% endwith %} From 825a4f4e4d704b5cee5e51960e17b887972ae56f Mon Sep 17 00:00:00 2001 From: Maryna Balioura Date: Sun, 3 Dec 2023 22:47:39 +0100 Subject: [PATCH 17/18] export/html: DIFF: update template type --- strictdoc/export/html/templates/screens/git/index.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/strictdoc/export/html/templates/screens/git/index.jinja b/strictdoc/export/html/templates/screens/git/index.jinja index dc1b38f48..e3897c0ad 100644 --- a/strictdoc/export/html/templates/screens/git/index.jinja +++ b/strictdoc/export/html/templates/screens/git/index.jinja @@ -1,5 +1,5 @@ {% extends "base.jinja.html" %} -{% set template_type = "Git" %} +{% set template_type = "DIFF" %} {% block head_css %} {{ super() }} From b7955f8e6e8a2452b8c9a052c0e065947a8da3a2 Mon Sep 17 00:00:00 2001 From: Stanislav Pankevich Date: Sun, 3 Dec 2023 22:29:57 +0100 Subject: [PATCH 18/18] git_client: speed up the rsync process by creating independent Git repos --- strictdoc/git/git_client.py | 73 ++++++++++++++++++++++++ strictdoc/git/project_diff_analyzer.py | 13 +++-- strictdoc/helpers/diff.py | 8 +-- strictdoc/server/app.py | 9 ++- strictdoc/server/routers/other_router.py | 69 ++++++++-------------- 5 files changed, 114 insertions(+), 58 deletions(-) diff --git a/strictdoc/git/git_client.py b/strictdoc/git/git_client.py index 0efe95db5..d2c246393 100644 --- a/strictdoc/git/git_client.py +++ b/strictdoc/git/git_client.py @@ -1,13 +1,86 @@ import os.path import subprocess +import tempfile +from pathlib import Path from typing import Optional +from strictdoc.helpers.timing import measure_performance + +PATH_TO_TMP_DIR = tempfile.gettempdir() +PATH_TO_SANDBOX_DIR = os.path.join( + PATH_TO_TMP_DIR, "strictdoc_cache", "git_sandbox" +) + class GitClient: def __init__(self, path_to_git_root: str): assert os.path.isdir(path_to_git_root) self.path_to_git_root: str = path_to_git_root + @staticmethod + def create_repo_from_local_copy(revision: str): + with measure_performance(f"Copy Git repo: {revision}"): + path_to_cwd = os.getcwd() + path_to_sandbox_git_repo = os.path.join( + PATH_TO_SANDBOX_DIR, revision + ) + path_to_sandbox_git_repo_git = os.path.join( + path_to_sandbox_git_repo, ".git" + ) + if revision != "HEAD+" and os.path.exists( + path_to_sandbox_git_repo_git + ): + git_client = GitClient(path_to_sandbox_git_repo) + + if git_client.is_clean_branch(): + return git_client + + Path(PATH_TO_SANDBOX_DIR).mkdir(parents=True, exist_ok=True) + + result = subprocess.run( + [ + "rsync", + "-ra", + "--delete", + "--exclude=.idea/", + "--exclude=.ruff_cache/", + "--exclude=__pycache__/", + "--exclude=build/", + "--exclude=output/", + "--exclude=strictdoc-project.github.io/", + ".", + path_to_sandbox_git_repo, + ], + cwd=path_to_cwd, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result + + git_client = GitClient(path_to_sandbox_git_repo) + + if revision != "HEAD+": + git_client.hard_reset(revision=revision) + git_client.clean() + + return git_client + + def is_clean_branch(self): + """ + https://unix.stackexchange.com/a/155077/77389 + """ + result = subprocess.run( + ["git", "status", "--porcelain"], + cwd=self.path_to_git_root, + capture_output=True, + text=True, + check=True, + ) + if result.returncode != 0: + return False + return result.stdout == "" + def add_file(self, path_to_file): result = subprocess.run( ["git", "add", path_to_file], diff --git a/strictdoc/git/project_diff_analyzer.py b/strictdoc/git/project_diff_analyzer.py index d06225dc8..2154cc299 100644 --- a/strictdoc/git/project_diff_analyzer.py +++ b/strictdoc/git/project_diff_analyzer.py @@ -437,13 +437,14 @@ def recurse(node): map_nodes_to_hashers[node].update(node_md5) return map_nodes_to_hashers[node].hexdigest().encode("utf-8") - # recurse(document) + # Keeping this code in case we will need to include child node hashes + # to parent node hashes recursively. This was the original + # implementation which we discarded. Now, each node's hash is + # self-contained. + # recurse(document) # noqa: ERA001 + for node_ in document_iterator.all_content(): - node_md5 = ( - map_nodes_to_hashers[node_] - .hexdigest() - .encode("utf-8") - ) + node_md5 = map_nodes_to_hashers[node_].hexdigest().encode("utf-8") map_nodes_to_hashers[document].update(node_md5) for node_, node_hasher_ in map_nodes_to_hashers.items(): diff --git a/strictdoc/helpers/diff.py b/strictdoc/helpers/diff.py index 6dab89958..1708ecad6 100644 --- a/strictdoc/helpers/diff.py +++ b/strictdoc/helpers/diff.py @@ -6,12 +6,8 @@ def similar(a, b): return SequenceMatcher(None, a, b).ratio() -red = ( - lambda text: f'{text}' -) -green = ( - lambda text: f'{text}' -) +red = lambda text: f'{text}' +green = lambda text: f'{text}' white = lambda text: f"{text}" diff --git a/strictdoc/server/app.py b/strictdoc/server/app.py index 7de24bc72..9f5148b2f 100644 --- a/strictdoc/server/app.py +++ b/strictdoc/server/app.py @@ -25,15 +25,20 @@ def create_app( ] # Uncomment this to enable performance measurements. - # @app.middleware("http") + @app.middleware("http") async def add_process_time_header( # pylint: disable=unused-variable request: Request, call_next ): start_time = time.time() response = await call_next(request) time_passed = round(time.time() - start_time, 3) + + request_path = request.url.path + if len(request.url.query) > 0: + request_path += f"?{request.url.query}" + print( # noqa: T201 - f"PERF: {request.method} {request.url} {time_passed}s" + f"PERF: {request.method} {request_path} {time_passed}s" ) return response diff --git a/strictdoc/server/routers/other_router.py b/strictdoc/server/routers/other_router.py index 920f55da8..4ceec4686 100644 --- a/strictdoc/server/routers/other_router.py +++ b/strictdoc/server/routers/other_router.py @@ -1,5 +1,4 @@ import os -import subprocess from copy import deepcopy from datetime import datetime from typing import Optional @@ -20,8 +19,7 @@ ProjectDiffAnalyzer, ProjectTreeDiffStats, ) -from strictdoc.helpers.parallelizer import NullParallelizer -from strictdoc.helpers.timing import measure_performance +from strictdoc.helpers.parallelizer import Parallelizer from strictdoc.server.routers.main_router import HTTP_STATUS_PRECONDITION_FAILED @@ -100,63 +98,46 @@ def get_git_diff( assert left_revision_resolved is not None assert right_revision_resolved is not None - path_to_cwd = os.getcwd() - path_to_sandbox_dir = "/tmp/sandbox" - - with measure_performance("RSYNC"): - result = subprocess.run( - [ - "rsync", - # "-vvraP", - "-r", - "--partial", - "--delete", - "--exclude=build/", - "--exclude=__pycache__/", - "--exclude=.ruff_cache/", - "--exclude=output/", - "--exclude=strictdoc-project.github.io/", - ".", - path_to_sandbox_dir, - ], - cwd=path_to_cwd, - capture_output=False, - text=True, - check=True, - ) - assert result.returncode == 0, result - - git_client = GitClient(path_to_sandbox_dir) - if right_revision_resolved != "HEAD+": - git_client.hard_reset(revision=right_revision_resolved) - git_client.clean() + git_client_lhs = GitClient.create_repo_from_local_copy( + left_revision_resolved + ) - parallelizer = NullParallelizer() + parallelizer = Parallelizer() - project_config_copy: ProjectConfig = deepcopy(project_config) - assert project_config_copy.export_input_paths is not None + project_config_copy_lhs: ProjectConfig = deepcopy(project_config) + assert project_config_copy_lhs.export_input_paths is not None export_input_rel_path = os.path.relpath( - project_config_copy.export_input_paths[0], os.getcwd() + project_config_copy_lhs.export_input_paths[0], os.getcwd() ) export_input_abs_path = os.path.join( - path_to_sandbox_dir, export_input_rel_path + git_client_lhs.path_to_git_root, export_input_rel_path ) - project_config_copy.export_input_paths = [export_input_abs_path] + project_config_copy_lhs.export_input_paths = [export_input_abs_path] traceability_index_rhs: TraceabilityIndex = ( TraceabilityIndexBuilder.create( - project_config=project_config_copy, + project_config=project_config_copy_lhs, parallelizer=parallelizer, ) ) - if left_revision_resolved != "HEAD+": - git_client.hard_reset(revision=left_revision_resolved) - git_client.clean() + git_client_rhs = GitClient.create_repo_from_local_copy( + right_revision_resolved + ) + + project_config_copy_rhs: ProjectConfig = deepcopy(project_config) + assert project_config_copy_rhs.export_input_paths is not None + export_input_rel_path = os.path.relpath( + project_config_copy_rhs.export_input_paths[0], os.getcwd() + ) + export_input_abs_path = os.path.join( + git_client_rhs.path_to_git_root, export_input_rel_path + ) + project_config_copy_rhs.export_input_paths = [export_input_abs_path] traceability_index_lhs: TraceabilityIndex = ( TraceabilityIndexBuilder.create( - project_config=project_config_copy, + project_config=project_config_copy_rhs, parallelizer=parallelizer, ) )