Skip to content

Commit

Permalink
refactor: refactor text work item generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@micha91
micha91 committed Sep 3, 2024
1 parent 0125dff commit 398a536
Show file tree
Hide file tree
Showing 7 changed files with 188 additions and 152 deletions.
20 changes: 6 additions & 14 deletions capella2polarion/connectors/polarion_worker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,7 @@

from capella2polarion import data_models
from capella2polarion.connectors import polarion_repo
from capella2polarion.converters import (
data_session,
document_config,
polarion_html_helper,
)
from capella2polarion.converters import data_session, polarion_html_helper

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -496,17 +492,13 @@ def _process_document_datas(self, client, document_datas):
for document_data in document_datas:
headings += document_data.headings
documents.append(document_data.document)
if document_data.text_work_items:
text_work_item_type = next(
iter(document_data.text_work_items.values())
).type
if document_data.text_work_item_provider.new_text_work_items:
self._create_and_update_text_work_items(
document_data.text_work_items, client
document_data.text_work_item_provider.new_text_work_items,
client,
)
polarion_html_helper.insert_text_work_items(
document_data.text_work_item_provider.insert_text_work_items(
document_data.document,
document_data.text_work_items,
text_work_item_type,
)
return documents, headings

Expand All @@ -526,7 +518,7 @@ def get_document(

def load_polarion_documents(
self,
document_infos: t.Iterable[document_config.DocumentInfo],
document_infos: t.Iterable[data_models.DocumentInfo],
) -> dict[
tuple[str | None, str, str],
tuple[polarion_api.Document | None, list[polarion_api.WorkItem]],
Expand Down
17 changes: 3 additions & 14 deletions capella2polarion/converters/document_config.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# Copyright DB InfraGO AG and contributors
# SPDX-License-Identifier: Apache-2.0
"""Module with classes and a loader for document rendering configs."""
import dataclasses
import logging
import pathlib
import typing as t
Expand All @@ -12,22 +11,12 @@
import pydantic
import yaml

from capella2polarion import data_models
from capella2polarion.converters import polarion_html_helper

logger = logging.getLogger(__name__)


@dataclasses.dataclass
class DocumentInfo:
"""Class for information regarding a document which should be created."""

project_id: str | None
module_folder: str
module_name: str
text_work_item_type: str
text_work_item_id_field: str


class WorkItemLayout(pydantic.BaseModel):
"""Configuration for rendering layouts of work items."""

Expand Down Expand Up @@ -93,11 +82,11 @@ class DocumentConfigs(pydantic.BaseModel):
pydantic.Field(default_factory=list)
)

def iterate_documents(self) -> t.Iterator[DocumentInfo]:
def iterate_documents(self) -> t.Iterator[data_models.DocumentInfo]:
"""Yield all document paths of the config as tuples."""
for conf in self.full_authority + self.mixed_authority:
for inst in conf.instances:
yield DocumentInfo(
yield data_models.DocumentInfo(
project_id=conf.project_id,
module_folder=inst.polarion_space,
module_name=inst.polarion_name,
Expand Down
142 changes: 33 additions & 109 deletions capella2polarion/converters/document_renderer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from .. import data_models
from . import document_config, polarion_html_helper
from . import text_work_item_provider as twi

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -169,10 +170,9 @@ def render_document(
heading_numbering: bool = False,
rendering_layouts: list[polarion_api.RenderingLayout] | None = None,
*,
text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD,
text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE,
text_work_item_provider: twi.TextWorkItemProvider | None = None,
**kwargs: t.Any,
):
) -> data_models.DocumentData:
"""Render a new Polarion document."""

@t.overload
Expand All @@ -182,11 +182,9 @@ def render_document(
template_name: str,
*,
document: polarion_api.Document,
text_work_items: dict[str, polarion_api.WorkItem],
text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD,
text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE,
text_work_item_provider: twi.TextWorkItemProvider | None = None,
**kwargs: t.Any,
):
) -> data_models.DocumentData:
"""Update an existing Polarion document."""

def render_document(
Expand All @@ -199,13 +197,13 @@ def render_document(
heading_numbering: bool = False,
rendering_layouts: list[polarion_api.RenderingLayout] | None = None,
document: polarion_api.Document | None = None,
text_work_items: dict[str, polarion_api.WorkItem] | None = None,
text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD,
text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE,
text_work_item_provider: twi.TextWorkItemProvider | None = None,
**kwargs: t.Any,
):
) -> data_models.DocumentData:
"""Render a Polarion document."""
text_work_items = text_work_items or {}
text_work_item_provider = (
text_work_item_provider or twi.TextWorkItemProvider()
)
if document is not None:
polarion_folder = document.module_folder
polarion_name = document.module_name
Expand Down Expand Up @@ -238,11 +236,8 @@ def render_document(
rendering_result = template.render(
model=self.model, session=session, **kwargs
)
new_text_work_items = self._extract_text_work_items(
text_work_item_provider.generate_text_work_items(
lxmlhtml.fragments_fromstring(rendering_result),
text_work_items,
text_work_item_type,
text_work_item_identifier,
)

document.home_page_content = polarion_api.TextContent(
Expand All @@ -252,7 +247,7 @@ def render_document(
document.rendering_layouts = session.rendering_layouts

return data_models.DocumentData(
document, session.headings, new_text_work_items
document, session.headings, text_work_item_provider
)

def update_mixed_authority_document(
Expand All @@ -262,11 +257,12 @@ def update_mixed_authority_document(
sections: dict[str, str],
global_parameters: dict[str, t.Any],
section_parameters: dict[str, dict[str, t.Any]],
text_work_items: dict[str, polarion_api.WorkItem],
text_work_item_identifier: str = polarion_html_helper.TEXT_WORK_ITEM_ID_FIELD,
text_work_item_type: str = polarion_html_helper.TEXT_WORK_ITEM_TYPE,
):
text_work_item_provider: twi.TextWorkItemProvider | None = None,
) -> data_models.DocumentData:
"""Update a mixed authority document."""
text_work_item_provider = (
text_work_item_provider or twi.TextWorkItemProvider()
)
assert (
document.home_page_content and document.home_page_content.value
), "In mixed authority the document must have content"
Expand All @@ -281,7 +277,6 @@ def update_mixed_authority_document(
env = self._get_jinja_env(template_folder)

new_content = []
new_text_work_items = {}
last_section_end = 0

for section_name, area in section_areas.items():
Expand Down Expand Up @@ -310,19 +305,9 @@ def update_mixed_authority_document(
work_item_ids = polarion_html_helper.extract_work_items(
current_content
)
section_text_work_items = {
text_id: work_item
for text_id, work_item in text_work_items.items()
if work_item.id in work_item_ids
}
html_fragments = lxmlhtml.fragments_fromstring(content)
new_text_work_items.update(
self._extract_text_work_items(
html_fragments,
section_text_work_items,
text_work_item_type,
text_work_item_identifier,
)
text_work_item_provider.generate_text_work_items(
html_fragments, work_item_ids
)
new_content += html_fragments

Expand All @@ -341,7 +326,7 @@ def update_mixed_authority_document(
document.rendering_layouts = session.rendering_layouts

return data_models.DocumentData(
document, session.headings, new_text_work_items
document, session.headings, text_work_item_provider
)

def _get_and_customize_doc(
Expand Down Expand Up @@ -387,25 +372,6 @@ def render_documents(

return self.projects

def _make_text_work_item_mapping(
self,
work_items: list[polarion_api.WorkItem],
text_work_item_field_id: str,
) -> dict[str, polarion_api.WorkItem]:
result = {}
for work_item in work_items:
# We only use those work items which have an ID defined by us
if text_id := work_item.additional_attributes.get(
text_work_item_field_id
):
if text_id in result:
raise ValueError(
f"There are multiple text work items with {text_work_item_field_id} == {text_id}"
)

result[text_id] = work_item
return result

def _check_document_status(
self,
document: polarion_api.Document,
Expand Down Expand Up @@ -448,6 +414,11 @@ def _render_mixed_authority_documents(
rendering_layouts,
config.heading_numbering,
)
text_work_item_provider = twi.TextWorkItemProvider(
config.text_work_item_id_field,
config.text_work_item_type,
text_work_items,
)
if old_doc is None:
logger.error(
"For document %s/%s no document was found, but it's "
Expand All @@ -467,11 +438,7 @@ def _render_mixed_authority_documents(
config.sections,
instance.params,
instance.section_params,
self._make_text_work_item_mapping(
text_work_items, config.text_work_item_id_field
),
config.text_work_item_id_field,
config.text_work_item_type,
text_work_item_provider,
)
except Exception as e:
logger.error(
Expand Down Expand Up @@ -505,6 +472,11 @@ def _render_full_authority_documents(
rendering_layouts,
config.heading_numbering,
)
text_work_item_provider = twi.TextWorkItemProvider(
config.text_work_item_id_field,
config.text_work_item_type,
text_work_items,
)
if old_doc:
if not self._check_document_status(old_doc, config):
continue
Expand All @@ -514,11 +486,7 @@ def _render_full_authority_documents(
config.template_directory,
config.template,
document=old_doc,
text_work_items=self._make_text_work_item_mapping(
text_work_items, config.text_work_item_id_field
),
text_work_item_identifier=config.text_work_item_id_field,
text_work_item_type=config.text_work_item_type,
text_work_item_provider=text_work_item_provider,
**instance.params,
)
except Exception as e:
Expand All @@ -542,8 +510,7 @@ def _render_full_authority_documents(
instance.polarion_title,
config.heading_numbering,
rendering_layouts,
text_work_item_identifier=config.text_work_item_id_field,
text_work_item_type=config.text_work_item_type,
text_work_item_provider=text_work_item_provider,
**instance.params,
)
except Exception as e:
Expand Down Expand Up @@ -603,46 +570,3 @@ def _extract_section_areas(self, html_elements: list[etree._Element]):
current_area_id = None
current_area_start = None
return section_areas

def _extract_text_work_items(
self,
content: list[lxmlhtml.HtmlElement],
text_work_items: dict[str, polarion_api.WorkItem],
text_work_item_type: str,
field_id: str,
) -> dict[str, polarion_api.WorkItem]:
work_items: dict[str, polarion_api.WorkItem] = {}
for element in content:
if element.tag != polarion_html_helper.WORK_ITEM_TAG:
continue

if not (text_id := element.get("id")):
raise ValueError("All work items must have an ID in template")

work_item = text_work_items.pop(
text_id,
polarion_api.WorkItem(
type=text_work_item_type,
title="",
status="open",
additional_attributes={field_id: text_id},
),
)
work_item.description_type = "text/html"
inner_content = "".join(
[
(
lxmlhtml.tostring(child, encoding="unicode")
if isinstance(child, lxmlhtml.HtmlElement)
else child
)
for child in element.iterchildren()
]
)
if element.text:
inner_content = element.text + inner_content

work_item.description = inner_content
work_items[text_id] = work_item

return work_items
Loading

0 comments on commit 398a536

Please sign in to comment.