From da6ae5805b42fa6e94449b27487e341977576f04 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Sat, 2 Sep 2023 10:04:07 +0200 Subject: [PATCH 01/13] wip(ci-templates)!: Install packages on special tags This is just for the staging pipeline. --- .gitlab-ci.yml | 1 + ci-templates/gitlab/synchronise_elements.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c217c946..f2dca510 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,7 @@ stages: .patch-pyproject-toml: &patch-pyproject-toml - sed -i -e 's/\(^ "polarion-rest-api-client\).*",/\1",/' pyproject.toml + - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl wheel: stage: build diff --git a/ci-templates/gitlab/synchronise_elements.yml b/ci-templates/gitlab/synchronise_elements.yml index a089ada8..bffb77dd 100644 --- a/ci-templates/gitlab/synchronise_elements.yml +++ b/ci-templates/gitlab/synchronise_elements.yml @@ -10,7 +10,8 @@ capella2polarion_synchronise_elements: artifacts: true script: - - pip install capella2polarion --pre + - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl + - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/capella2polarion/less-requests/capella2polarion-less_requests-py3-none-any.whl - > python \ -m capella2polarion \ From 80ca21b053c51395ad5356c4a344563fa424e236 Mon Sep 17 00:00:00 2001 From: Michael Harbarth Date: Tue, 12 Sep 2023 16:49:48 +0200 Subject: [PATCH 02/13] fix: don't use get_linked_text for conditions as it does not work properly for deleted references --- capella2polarion/elements/serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 13b73a96..3a50d308 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -186,7 +186,7 @@ def include_pre_and_post_condition( def get_condition(cap: PrePostConditionElement, name: str) -> str: if not (condition := getattr(cap, name)): return "" - return get_linked_text(condition, ctx) + return condition.specification["capella:linkedText"].striptags() def strike_through(string: str) -> str: if match := RE_DESCR_DELETED_PATTERN.match(string): From 91996a473cfdbf34285155034cfe3e8533c4a164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernst=20W=C3=BCrger?= Date: Wed, 13 Sep 2023 13:21:07 +0200 Subject: [PATCH 03/13] fix: Prevent infinite SystemActor creation Picked changes from b09ce597. --- capella2polarion/elements/element.py | 20 +++++++++++++++++--- tests/test_elements.py | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index 516409c7..fe39b2ee 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -13,7 +13,7 @@ from capellambse.model import diagram as diag from capella2polarion import elements -from capella2polarion.elements import serialize +from capella2polarion.elements import helpers, serialize logger = logging.getLogger(__name__) @@ -28,11 +28,25 @@ def create_work_items( ) -> list[serialize.CapellaWorkItem]: """Create a set of work items in Polarion.""" objects = chain.from_iterable(ctx["ELEMENTS"].values()) - work_items = [ + _work_items = [ serialize.element(obj, ctx, serialize.generic_work_item) for obj in objects ] - return list(filter(None.__ne__, work_items)) # type: ignore[arg-type] + _work_items = list(filter(None.__ne__, _work_items)) + valid_types = set(map(helpers.resolve_element_type, set(ctx["ELEMENTS"]))) + work_items: list[polarion_api.CapellaWorkItem] = [] + missing_types: set[str] = set() + for work_item in _work_items: + assert work_item is not None + if work_item.type in valid_types: + work_items.append(work_item) + else: + missing_types.add(work_item.type) + logger.debug( + "%r are missing in the capella2polarion configuration", + ", ".join(missing_types), + ) + return work_items def create_links( diff --git a/tests/test_elements.py b/tests/test_elements.py index c0502c22..d83b2954 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -192,6 +192,8 @@ def test_create_work_items( monkeypatch: pytest.MonkeyPatch, context: dict[str, t.Any] ): del context["ELEMENTS"]["UnsupportedFakeModelObject"] + context["MODEL"] = model = mock.MagicMock() + model.by_uuid.side_effect = context["ELEMENTS"]["FakeModelObject"] monkeypatch.setattr( serialize, "generic_work_item", From 083065b221a72372290441a43ec7229d1b775dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ernst=20W=C3=BCrger?= <50786483+ewuerger@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:59:57 +0200 Subject: [PATCH 04/13] fix: Apply suggestions from code review Co-authored-by: micha91 --- capella2polarion/elements/__init__.py | 12 ++++++------ capella2polarion/elements/serialize.py | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/capella2polarion/elements/__init__.py b/capella2polarion/elements/__init__.py index c08f97ff..c8b161f5 100644 --- a/capella2polarion/elements/__init__.py +++ b/capella2polarion/elements/__init__.py @@ -103,13 +103,13 @@ def post_work_items(ctx: dict[str, t.Any]) -> None: ctx The context for the workitem operation to be processed. """ - work_items = [ - wi - for wi in ctx["WORK_ITEMS"].values() - if wi.uuid_capella not in ctx["POLARION_ID_MAP"] - ] - for work_item in work_items: + work_items: list[serialize.CapellaWorkItem] = [] + for work_item in ctx["WORK_ITEMS"].values(): + if work_item.uuid_capella not in ctx["POLARION_ID_MAP"]: + continue + assert work_item is not None + work_items.append(work_item) logger.info("Create work item for %r...", work_item.title) if work_items: try: diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 3a50d308..674c3670 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -48,7 +48,6 @@ class Condition(t.TypedDict): uuid_capella: str | None preCondition: Condition | None postCondition: Condition | None - checksum: str | None def element( From 39475d5bca9f572cf5fe48e20de01862c79d49d9 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Wed, 11 Oct 2023 16:33:45 +0200 Subject: [PATCH 05/13] feat(model-elements): Add `input|output_exchanges` link handler --- capella2polarion/elements/element.py | 41 ++++++++++++++++++++++---- capella2polarion/elements/serialize.py | 8 +++-- tests/test_elements.py | 24 +++++++++++++-- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index fe39b2ee..46b831e6 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -4,6 +4,7 @@ from __future__ import annotations import collections.abc as cabc +import functools import logging import typing as t from itertools import chain @@ -11,6 +12,7 @@ import polarion_rest_api_client as polarion_api from capellambse.model import common from capellambse.model import diagram as diag +from capellambse.model.crosslayer import fa from capella2polarion import elements from capella2polarion.elements import helpers, serialize @@ -80,7 +82,7 @@ def create_links( continue if isinstance(refs, common.ElementList): - new = refs.by_uuid + new: cabc.Iterable[str] = refs.by_uuid # type: ignore[assignment] else: assert hasattr(refs, "uuid") new = [refs.uuid] @@ -141,7 +143,9 @@ def _handle_diagram_reference_links( return ref_links -def _collect_uuids(nodes: list[common.GenericElement]) -> cabc.Iterator[str]: +def _collect_uuids( + nodes: cabc.Iterable[common.GenericElement], +) -> cabc.Iterator[str]: type_resolvers = TYPE_RESOLVERS for node in nodes: uuid = node.uuid @@ -171,7 +175,34 @@ def _create( return list(filter(None.__ne__, _new_links)) -CUSTOM_LINKS = { - "description_reference": _handle_description_reference_links, - "diagram_elements": _handle_diagram_reference_links, +def _handle_exchanges( + context: dict[str, t.Any], + obj: fa.Function, + role_id: str, + links: dict[str, polarion_api.WorkItemLink], + attr: str = "inputs", +) -> list[polarion_api.WorkItemLink]: + wid = context["POLARION_ID_MAP"][obj.uuid] + exchanges: list[str] = [] + for element in getattr(obj, attr): + uuids = element.exchanges.by_uuid + exs = _get_work_item_ids(context, wid, uuids, role_id) + exchanges.extend(set(exs)) + return _create(context, wid, role_id, exchanges, links) + + +CustomLinkMaker = cabc.Callable[ + [ + dict[str, t.Any], + diag.Diagram | common.GenericElement, + str, + dict[str, t.Any], + ], + list[polarion_api.WorkItemLink], +] +CUSTOM_LINKS: dict[str, CustomLinkMaker] = { + "description_reference": _handle_description_reference_links, # type: ignore[dict-item] + "diagram_elements": _handle_diagram_reference_links, # type: ignore[dict-item] + "input_exchanges": functools.partial(_handle_exchanges, attr="inputs"), + "output_exchanges": functools.partial(_handle_exchanges, attr="outputs"), } diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 674c3670..5c01e6cd 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -173,8 +173,7 @@ def replace_markup( f'id="fake" data-item-id="{pid}" data-option-id="long">' "" ) - else: - return non_matcher(match.group(0)) + return non_matcher(match.group(0)) def include_pre_and_post_condition( @@ -262,7 +261,10 @@ def physical_component( return work_item -SERIALIZERS = { +Serializer = cabc.Callable[ + [common.GenericElement, dict[str, t.Any]], CapellaWorkItem +] +SERIALIZERS: dict[str, Serializer] = { "CapabilityRealization": include_pre_and_post_condition, "LogicalComponent": component_or_actor, "OperationalCapability": include_pre_and_post_condition, diff --git a/tests/test_elements.py b/tests/test_elements.py index d83b2954..4a070740 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -137,7 +137,7 @@ def __init__( self, uuid: str, name: str = "", - attribute: FakeModelObject | None = None, + attribute: t.Any | None = None, ): self.uuid = uuid self.name = name @@ -237,6 +237,26 @@ def test_create_links_custom_resolver(context: dict[str, t.Any]): assert links == [expected] + @staticmethod + def test_create_links_custom_exchanges_resolver(context: dict[str, t.Any]): + function_uuid = "ceffa011-7b66-4b3c-9885-8e075e312ffa" + obj = context["MODEL"].by_uuid(function_uuid) + context["POLARION_ID_MAP"][function_uuid] = "Obj-1" + context["POLARION_ID_MAP"][ + "1a414995-f4cd-488c-8152-486e459fb9de" + ] = "Obj-2" + context["ROLES"] = {"SystemFunction": ["input_exchanges"]} + expected = polarion_api.WorkItemLink( + "Obj-1", + "Obj-2", + "input_exchanges", + secondary_work_item_project="project_id", + ) + + links = element.create_links(obj, context) + + assert links == [expected] + @staticmethod def test_create_links_missing_attribute( context: dict[str, t.Any], caplog: pytest.LogCaptureFixture @@ -282,7 +302,7 @@ def test_create_links_from_ElementList(context: dict[str, t.Any]): secondary_work_item_project="project_id", ) - links = element.create_links(obj, context) + links = element.create_links(obj, context) # type: ignore[arg-type] assert expected_link in links assert expected_link1 in links From da6a5bc2c8eef9d0099207ce7a6f2ddbd6eabe03 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Wed, 11 Oct 2023 17:04:35 +0200 Subject: [PATCH 06/13] fix!: Fix `post_work_items` --- capella2polarion/elements/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capella2polarion/elements/__init__.py b/capella2polarion/elements/__init__.py index c8b161f5..a2899692 100644 --- a/capella2polarion/elements/__init__.py +++ b/capella2polarion/elements/__init__.py @@ -105,7 +105,7 @@ def post_work_items(ctx: dict[str, t.Any]) -> None: """ work_items: list[serialize.CapellaWorkItem] = [] for work_item in ctx["WORK_ITEMS"].values(): - if work_item.uuid_capella not in ctx["POLARION_ID_MAP"]: + if work_item.uuid_capella in ctx["POLARION_ID_MAP"]: continue assert work_item is not None From c7e88ea61339a0198fc1f2785251121bec79f8f5 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Thu, 19 Oct 2023 16:38:30 +0200 Subject: [PATCH 07/13] fix: Fix various bugs - Fix debug log message for empty missing types - Merge diagram work item creator into generic work item creator - Handle all roles from the wildcard entry in the config. Also include `Diagram`. --- capella2polarion/__main__.py | 63 ++++++++++++++++++++++----- capella2polarion/elements/__init__.py | 18 ++++---- capella2polarion/elements/diagram.py | 24 ---------- capella2polarion/elements/element.py | 29 ++++++++---- tests/data/model_elements/config.yaml | 15 ++++--- tests/test_cli.py | 5 --- tests/test_elements.py | 8 ++-- 7 files changed, 94 insertions(+), 68 deletions(-) delete mode 100644 capella2polarion/elements/diagram.py diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 46bd34a0..097ade32 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -79,13 +79,30 @@ def _get_roles_from_config(ctx: dict[str, t.Any]) -> dict[str, list[str]]: roles[key] = list(role_ids) else: roles[typ] = [] - roles["Diagram"] = ["diagram_elements"] return roles def _sanitize_config( - config: dict[str, list[str | dict[str, t.Any]]], special: dict[str, t.Any] + config: dict[str, list[str | dict[str, t.Any]]], + special: list[str | dict[str, t.Any]], ) -> dict[str, t.Any]: + special_config: dict[str, t.Any] = {} + for typ in special: + if isinstance(typ, str): + special_config[typ] = None + else: + special_config.update(typ) + + lookup: dict[str, dict[str, list[str]]] = {} + for layer, xtypes in config.items(): + for xt in xtypes: + if isinstance(xt, str): + item: dict[str, list[str]] = {xt: []} + else: + item = xt + + lookup.setdefault(layer, {}).update(item) + new_config: dict[str, t.Any] = {} for layer, xtypes in config.items(): new_entries: list[str | dict[str, t.Any]] = [] @@ -93,16 +110,36 @@ def _sanitize_config( if isinstance(xtype, dict): for sub_key, sub_value in xtype.items(): new_value = ( - special.get("*", []) - + special.get(sub_key, []) + special_config.get("*", []) + + special_config.get(sub_key, []) + sub_value ) new_entries.append({sub_key: new_value}) else: - if new_value := special.get("*", []) + special.get(xtype, []): + star = special_config.get("*", []) + special_xtype = special_config.get(xtype, []) + if new_value := star + special_xtype: new_entries.append({xtype: new_value}) else: new_entries.append(xtype) + + wildcard_values = special_config.get("*", []) + for key, value in special_config.items(): + if key == "*": + continue + + if isinstance(value, list): + new_value = ( + lookup.get(layer, {}).get(key, []) + + wildcard_values + + value + ) + new_entries.append({key: new_value}) + elif value is None and key not in [ + entry if isinstance(entry, str) else list(entry.keys())[0] + for entry in new_entries + ]: + new_entries.append({key: wildcard_values}) new_config[layer] = new_entries return new_config @@ -189,16 +226,20 @@ def model_elements( ctx.obj["POLARION_ID_MAP"] = { uuid: wi.id for uuid, wi in ctx.obj["POLARION_WI_MAP"].items() } - diagrams = ctx.obj["ELEMENTS"].pop("Diagram", []) - work_items = elements.element.create_work_items(ctx.obj) - ctx.obj["ELEMENTS"]["Diagram"] = diagrams - pdiagrams = elements.diagram.create_diagrams(ctx.obj) - ctx.obj["WORK_ITEMS"] = { - wi.uuid_capella: wi for wi in work_items + pdiagrams + duuids = { + diag["uuid"] for diag in ctx.obj["DIAGRAM_IDX"] if diag["success"] } + ctx.obj["ELEMENTS"]["Diagram"] = [ + diag for diag in ctx.obj["ELEMENTS"]["Diagram"] if diag.uuid in duuids + ] + work_items = elements.element.create_work_items(ctx.obj) + ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} elements.delete_work_items(ctx.obj) elements.post_work_items(ctx.obj) + + work_items = elements.element.create_work_items(ctx.obj) + ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} elements.patch_work_items(ctx.obj) elements.make_model_elements_index(ctx.obj) diff --git a/capella2polarion/elements/__init__.py b/capella2polarion/elements/__init__.py index a2899692..ab9c208a 100644 --- a/capella2polarion/elements/__init__.py +++ b/capella2polarion/elements/__init__.py @@ -114,6 +114,11 @@ def post_work_items(ctx: dict[str, t.Any]) -> None: if work_items: try: ctx["API"].create_work_items(work_items) + workitems = {wi.uuid_capella: wi for wi in work_items if wi.id} + ctx["POLARION_WI_MAP"].update(workitems) + ctx["POLARION_ID_MAP"] = { + uuid: wi.id for uuid, wi in ctx["POLARION_WI_MAP"].items() + } except polarion_api.PolarionApiException as error: logger.error("Creating work items failed. %s", error.args[0]) @@ -140,21 +145,16 @@ def add_content( setattr(work_item, key, value) return work_item - ctx["POLARION_WI_MAP"] = get_polarion_wi_map(ctx) ctx["POLARION_ID_MAP"] = uuids = { uuid: wi.id for uuid, wi in ctx["POLARION_WI_MAP"].items() - if wi.status == "open" and uuid in ctx["WORK_ITEMS"] + if wi.status == "open" and wi.uuid_capella and wi.id } for uuid in uuids: elements = ctx["MODEL"] if uuid.startswith("_"): elements = ctx["MODEL"].diagrams - try: - obj = elements.by_uuid(uuid) - except KeyError: - logger.error("Weird %r", uuid) - continue + obj = elements.by_uuid(uuid) links = element.create_links(obj, ctx) @@ -189,6 +189,9 @@ def get_elements_and_type_map( if isinstance(typ, dict): typ = list(typ.keys())[0] + if typ == "Diagram": + continue + xtype = convert_type.get(typ, typ) objects = ctx["MODEL"].search(xtype, below=below) elements.setdefault(typ, []).extend(objects) @@ -277,7 +280,6 @@ def make_model_elements_index(ctx: dict[str, t.Any]) -> None: from . import ( # pylint: disable=cyclic-import api_helper, - diagram, element, helpers, serialize, diff --git a/capella2polarion/elements/diagram.py b/capella2polarion/elements/diagram.py deleted file mode 100644 index 4c82436e..00000000 --- a/capella2polarion/elements/diagram.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright DB Netz AG and contributors -# SPDX-License-Identifier: Apache-2.0 -"""Objects for synchronization of Capella diagrams to polarion.""" -from __future__ import annotations - -import logging -import typing as t - -from capella2polarion.elements import serialize - -logger = logging.getLogger(__name__) - - -def create_diagrams(ctx: dict[str, t.Any]) -> list[serialize.CapellaWorkItem]: - """Return a set of new work items of type ``diagram``.""" - uuids = {diag["uuid"] for diag in ctx["DIAGRAM_IDX"] if diag["success"]} - diagrams = [ - diag for diag in ctx["ELEMENTS"]["Diagram"] if diag.uuid in uuids - ] - work_items = [ - serialize.element(diagram, ctx, serialize.diagram) - for diagram in diagrams - ] - return list(filter(None.__ne__, work_items)) # type:ignore[arg-type] diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index 46b831e6..84ae08d5 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -28,12 +28,21 @@ def create_work_items( ctx: dict[str, t.Any] ) -> list[serialize.CapellaWorkItem]: - """Create a set of work items in Polarion.""" + """Create a list of work items for Polarion.""" objects = chain.from_iterable(ctx["ELEMENTS"].values()) - _work_items = [ - serialize.element(obj, ctx, serialize.generic_work_item) - for obj in objects + _work_items = [] + serializer: cabc.Callable[ + [diag.Diagram | common.GenericElement, dict[str, t.Any]], + serialize.CapellaWorkItem, ] + for obj in objects: + if isinstance(obj, diag.Diagram): + serializer = serialize.diagram + else: + serializer = serialize.generic_work_item + + _work_items.append(serialize.element(obj, ctx, serializer)) + _work_items = list(filter(None.__ne__, _work_items)) valid_types = set(map(helpers.resolve_element_type, set(ctx["ELEMENTS"]))) work_items: list[polarion_api.CapellaWorkItem] = [] @@ -44,10 +53,12 @@ def create_work_items( work_items.append(work_item) else: missing_types.add(work_item.type) - logger.debug( - "%r are missing in the capella2polarion configuration", - ", ".join(missing_types), - ) + + if missing_types: + logger.debug( + "%r are missing in the capella2polarion configuration", + ", ".join(missing_types), + ) return work_items @@ -118,7 +129,7 @@ def _handle_description_reference_links( role_id: str, links: dict[str, polarion_api.WorkItemLink], ) -> list[polarion_api.WorkItemLink]: - refs = context["DESCR_REFERENCES"].get(obj.uuid) + refs = context["DESCR_REFERENCES"].get(obj.uuid, []) wid = context["POLARION_ID_MAP"][obj.uuid] refs = set(_get_work_item_ids(context, wid, refs, role_id)) return _create(context, wid, role_id, refs, links) diff --git a/tests/data/model_elements/config.yaml b/tests/data/model_elements/config.yaml index e8647854..e032dcbb 100644 --- a/tests/data/model_elements/config.yaml +++ b/tests/data/model_elements/config.yaml @@ -2,11 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 "*": # All layers - "*": # All class types - - parent # Specify workitem links - - description_reference # Custom attribute - Class: - - state_machines + - "*": # All class types + - parent # Specify workitem links + - description_reference # Custom attribute + - Class: + - state_machines + - Diagram: + - diagram_elements + - Constraint oa: # Specify below - OperationalCapability: # Capella Type with references @@ -19,7 +22,6 @@ oa: # Specify below - CommunicationMean - Class - StateMachine - - Constraint sa: - SystemComponent: @@ -34,7 +36,6 @@ sa: - exchanged_items - ExchangeItem - Class - - Constraint pa: - PhysicalComponent: diff --git a/tests/test_cli.py b/tests/test_cli.py index a766ff82..0b752e17 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -67,10 +67,6 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): {}, ), ) - mock_create_diagrams = mock.MagicMock() - monkeypatch.setattr( - elements.diagram, "create_diagrams", mock_create_diagrams - ) mock_delete_work_items = mock.MagicMock() monkeypatch.setattr(elements, "delete_work_items", mock_delete_work_items) mock_post_work_items = mock.MagicMock() @@ -93,5 +89,4 @@ def test_migrate_model_elements(monkeypatch: pytest.MonkeyPatch): assert mock_delete_work_items.call_count == 1 assert mock_patch_work_items.call_count == 1 assert mock_post_work_items.call_count == 1 - assert mock_create_diagrams.call_count == 1 assert ELEMENTS_IDX_PATH.exists() diff --git a/tests/test_elements.py b/tests/test_elements.py index 4a070740..de07e915 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -14,7 +14,7 @@ from capellambse.model import common from capella2polarion import elements -from capella2polarion.elements import diagram, element, helpers, serialize +from capella2polarion.elements import element, helpers, serialize # pylint: disable-next=relative-beyond-top-level, useless-suppression from .conftest import TEST_DIAGRAM_CACHE, TEST_HOST # type: ignore[import] @@ -90,7 +90,7 @@ def context( def test_create_diagrams(context: dict[str, t.Any]): context["ELEMENTS"] = {"Diagram": context["MODEL"].diagrams} - diagrams = diagram.create_diagrams(context) + diagrams = element.create_work_items(context) assert len(diagrams) == 1 work_item = diagrams[0] @@ -116,7 +116,7 @@ def test_create_diagrams_filters_non_diagram_elements( attributes.return_value = None monkeypatch.setattr(serialize, "element", attributes) - diagram.create_diagrams(context) + element.create_work_items(context) assert context["API"].create_work_items.call_count == 0 @@ -385,7 +385,7 @@ def test_update_work_items_filters_work_items_with_same_checksum( @staticmethod def test_update_links_with_no_elements(context: dict[str, t.Any]): - context["POLARION_ID_MAP"] = {} + context["POLARION_WI_MAP"] = {} elements.patch_work_items(context) From ece1cde8489c6eff36caaab307d0eb0801f2e577 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Thu, 19 Oct 2023 16:47:08 +0200 Subject: [PATCH 08/13] refactor: Simplify high level calls --- capella2polarion/__main__.py | 7 +++---- capella2polarion/elements/element.py | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/capella2polarion/__main__.py b/capella2polarion/__main__.py index 097ade32..23e0f2e8 100644 --- a/capella2polarion/__main__.py +++ b/capella2polarion/__main__.py @@ -232,14 +232,13 @@ def model_elements( ctx.obj["ELEMENTS"]["Diagram"] = [ diag for diag in ctx.obj["ELEMENTS"]["Diagram"] if diag.uuid in duuids ] - work_items = elements.element.create_work_items(ctx.obj) - ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} + elements.element.create_work_items(ctx.obj) elements.delete_work_items(ctx.obj) elements.post_work_items(ctx.obj) - work_items = elements.element.create_work_items(ctx.obj) - ctx.obj["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} + # Create missing links b/c of unresolved references + elements.element.create_work_items(ctx.obj) elements.patch_work_items(ctx.obj) elements.make_model_elements_index(ctx.obj) diff --git a/capella2polarion/elements/element.py b/capella2polarion/elements/element.py index 84ae08d5..c64ef780 100644 --- a/capella2polarion/elements/element.py +++ b/capella2polarion/elements/element.py @@ -59,6 +59,7 @@ def create_work_items( "%r are missing in the capella2polarion configuration", ", ".join(missing_types), ) + ctx["WORK_ITEMS"] = {wi.uuid_capella: wi for wi in work_items} return work_items @@ -212,8 +213,8 @@ def _handle_exchanges( list[polarion_api.WorkItemLink], ] CUSTOM_LINKS: dict[str, CustomLinkMaker] = { - "description_reference": _handle_description_reference_links, # type: ignore[dict-item] - "diagram_elements": _handle_diagram_reference_links, # type: ignore[dict-item] + "description_reference": _handle_description_reference_links, + "diagram_elements": _handle_diagram_reference_links, "input_exchanges": functools.partial(_handle_exchanges, attr="inputs"), "output_exchanges": functools.partial(_handle_exchanges, attr="outputs"), } From 6136835c2294097cb1f289d2da58c1aaf681dc1f Mon Sep 17 00:00:00 2001 From: ewuerger Date: Thu, 19 Oct 2023 16:56:05 +0200 Subject: [PATCH 09/13] fix: Prepare for production pipelines --- .gitlab-ci.yml | 1 - ci-templates/gitlab/synchronise_elements.yml | 3 +-- pyproject.toml | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f2dca510..c217c946 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,6 @@ stages: .patch-pyproject-toml: &patch-pyproject-toml - sed -i -e 's/\(^ "polarion-rest-api-client\).*",/\1",/' pyproject.toml - - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl wheel: stage: build diff --git a/ci-templates/gitlab/synchronise_elements.yml b/ci-templates/gitlab/synchronise_elements.yml index bffb77dd..a089ada8 100644 --- a/ci-templates/gitlab/synchronise_elements.yml +++ b/ci-templates/gitlab/synchronise_elements.yml @@ -10,8 +10,7 @@ capella2polarion_synchronise_elements: artifacts: true script: - - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/polarion-rest-api-client/attachments/polarion_rest_api_client-attachments-py3-none-any.whl - - pip install https://$PYPI_ARTIFACTORY_USERNAME:$PYPI_ARTIFACTORY_PASSWORD@$PYPI_ARTIFACTORY/capella2polarion/less-requests/capella2polarion-less_requests-py3-none-any.whl + - pip install capella2polarion --pre - > python \ -m capella2polarion \ diff --git a/pyproject.toml b/pyproject.toml index b9a600b9..0bb414cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "capellambse", "click", "PyYAML", - "polarion-rest-api-client @ git+https://github.com/DSD-DBS/polarion-rest-api-client.git@feat-add-attachments-workitem-relations", + "polarion-rest-api-client @ git+https://github.com/DSD-DBS/polarion-rest-api-client.git@v0.2.1", "requests", ] From fcf080f235a26aa57471d5d9c5ca691bb28f5a7c Mon Sep 17 00:00:00 2001 From: ewuerger Date: Fri, 20 Oct 2023 13:34:09 +0200 Subject: [PATCH 10/13] feat: Add requirement types custom fields to generic serializer --- capella2polarion/elements/serialize.py | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 5c01e6cd..210cd525 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -4,6 +4,7 @@ from __future__ import annotations import base64 +import collections import collections.abc as cabc import logging import mimetypes @@ -113,6 +114,7 @@ def _generic_work_item( raw_description = getattr(obj, "description", markupsafe.Markup("")) uuids, value = _sanitize_description(raw_description, ctx) ctx.setdefault("DESCR_REFERENCES", {})[obj.uuid] = uuids + requirement_types = _get_requirement_types_text(obj) return CapellaWorkItem( type=helpers.resolve_element_type(xtype), title=obj.name, @@ -120,9 +122,45 @@ def _generic_work_item( description=value, status="open", uuid_capella=obj.uuid, + **requirement_types, ) +def _get_requirement_types_text( + obj: common.GenericElement, +) -> dict[str, dict[str, str]]: + type_texts = collections.defaultdict(list) + for req in obj.requirements: + if req is None: + logger.error( + "RequirementsRelation with broken target found %r", obj.name + ) + continue + + if not (req.type and req.text): + continue + + type_texts[req.type.long_name].append(req.text) + return _format_texts(type_texts) + + +def _format_texts( + type_texts: dict[str, list[str]] +) -> dict[str, dict[str, str]]: + def _format(texts: list[str]) -> dict[str, str]: + if len(texts) > 1: + items = "".join(f"
  • {text}
    • " for text in texts) + text = f"
      {items}
    " + else: + text = texts[0] + return {"type": "text/html", "value": text} + + requirement_types = {} + for typ, texts in type_texts.items(): + requirement_types[typ.lower()] = _format(texts) + return requirement_types + + def _sanitize_description( descr: markupsafe.Markup, ctx: dict[str, t.Any] ) -> tuple[list[str], markupsafe.Markup]: From 60b237ec3e125dadc3c6b5e5458382dc9afe52f5 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Mon, 23 Oct 2023 16:15:36 +0200 Subject: [PATCH 11/13] test: Fix test with requirements field --- capella2polarion/elements/serialize.py | 3 +++ tests/test_elements.py | 37 +++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 210cd525..51f73534 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -138,6 +138,9 @@ def _get_requirement_types_text( continue if not (req.type and req.text): + logger.warning( + "Requirement without text or type found %r", req.name + ) continue type_texts[req.type.long_name].append(req.text) diff --git a/tests/test_elements.py b/tests/test_elements.py index de07e915..7d0f5800 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -62,6 +62,12 @@ TEST_WI_CHECKSUM = ( "73508ec0c3048c5b33316dfa56ef5e5f4179ff69efaa209e47ab65b111415e82" ) +TEST_REQ_TEXT = ( + "

    Test requirement 1 really l o n g text that is way too long to " + "display here as that

    \n\n

    < > " '

    \n\n
      \n\t
    • " + "This is a list
      • \n\t
    • an unordered one
      • \n
    \n\n
      \n\t" + "
    1. Ordered list
      2. \n\t
    2. Ok
      3. \n
    \n" +) class TestDiagramElements: @@ -478,7 +484,7 @@ def test__decode_diagram(): @pytest.mark.parametrize( "uuid,expected", [ - ( + pytest.param( TEST_ELEMENT_UUID, { "type": "logicalComponent", @@ -486,9 +492,14 @@ def test__decode_diagram(): "uuid_capella": TEST_ELEMENT_UUID, "description_type": "text/html", "description": markupsafe.Markup(TEST_DESCR), + "reqtype": { + "type": "text/html", + "value": markupsafe.Markup(TEST_REQ_TEXT), + }, }, + id="logicalComponent", ), - ( + pytest.param( TEST_OCAP_UUID, { "type": "operationalCapability", @@ -501,8 +512,9 @@ def test__decode_diagram(): "postCondition": {"type": "text/html", "value": ""}, }, }, + id="operationalCapability", ), - ( + pytest.param( TEST_WE_UUID, { "type": "entity", @@ -511,8 +523,9 @@ def test__decode_diagram(): "description_type": "text/html", "description": markupsafe.Markup(TEST_WE_DESCR), }, + id="entity", ), - ( + pytest.param( TEST_ACTOR_UUID, { "type": "logicalActor", @@ -524,8 +537,9 @@ def test__decode_diagram(): "and greatest mage of all time.

    \n" ), }, + id="logicalActor", ), - ( + pytest.param( TEST_PHYS_COMP, { "type": "physicalComponent", @@ -534,8 +548,9 @@ def test__decode_diagram(): "description_type": "text/html", "description": markupsafe.Markup(""), }, + id="physicalComponent", ), - ( + pytest.param( TEST_PHYS_NODE, { "type": "physicalComponentNode", @@ -544,8 +559,9 @@ def test__decode_diagram(): "description_type": "text/html", "description": markupsafe.Markup(""), }, + id="physicalComponentNode", ), - ( + pytest.param( TEST_SCENARIO, { "type": "scenario", @@ -558,8 +574,9 @@ def test__decode_diagram(): "postCondition": {"type": "text/html", "value": ""}, }, }, + id="scenario", ), - ( + pytest.param( TEST_CAP_REAL, { "type": "capabilityRealization", @@ -572,8 +589,9 @@ def test__decode_diagram(): "postCondition": {"type": "text/html", "value": ""}, }, }, + id="capabilityRealization", ), - ( + pytest.param( TEST_CONSTRAINT, { "type": "constraint", @@ -584,6 +602,7 @@ def test__decode_diagram(): "This is a test context.Make Food" ), }, + id="constraint", ), ], ) From 9f78cf2345da2b8e1b0860902c318bd1c41aa319 Mon Sep 17 00:00:00 2001 From: ewuerger Date: Fri, 20 Oct 2023 13:34:09 +0200 Subject: [PATCH 12/13] feat: Add requirement types custom fields to generic serializer --- capella2polarion/elements/serialize.py | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 6a0bfcf8..1f192799 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -4,6 +4,7 @@ from __future__ import annotations import base64 +import collections import collections.abc as cabc import logging import mimetypes @@ -113,6 +114,7 @@ def _generic_work_item( raw_description = getattr(obj, "description", markupsafe.Markup("")) uuids, value = _sanitize_description(obj, raw_description, ctx) ctx.setdefault("DESCR_REFERENCES", {})[obj.uuid] = uuids + requirement_types = _get_requirement_types_text(obj) return CapellaWorkItem( type=helpers.resolve_element_type(xtype), title=obj.name, @@ -120,9 +122,45 @@ def _generic_work_item( description=value, status="open", uuid_capella=obj.uuid, + **requirement_types, ) +def _get_requirement_types_text( + obj: common.GenericElement, +) -> dict[str, dict[str, str]]: + type_texts = collections.defaultdict(list) + for req in obj.requirements: + if req is None: + logger.error( + "RequirementsRelation with broken target found %r", obj.name + ) + continue + + if not (req.type and req.text): + continue + + type_texts[req.type.long_name].append(req.text) + return _format_texts(type_texts) + + +def _format_texts( + type_texts: dict[str, list[str]] +) -> dict[str, dict[str, str]]: + def _format(texts: list[str]) -> dict[str, str]: + if len(texts) > 1: + items = "".join(f"
  • {text}
    • " for text in texts) + text = f"
      {items}
    " + else: + text = texts[0] + return {"type": "text/html", "value": text} + + requirement_types = {} + for typ, texts in type_texts.items(): + requirement_types[typ.lower()] = _format(texts) + return requirement_types + + def _sanitize_description( obj: common.GenericElement, descr: markupsafe.Markup, ctx: dict[str, t.Any] ) -> tuple[list[str], markupsafe.Markup]: From 2a3d26b432d7439d07c574c398b3c829bbdd3f0c Mon Sep 17 00:00:00 2001 From: ewuerger Date: Mon, 23 Oct 2023 16:15:36 +0200 Subject: [PATCH 13/13] test: Fix test with requirements field --- capella2polarion/elements/serialize.py | 3 +++ tests/test_elements.py | 37 +++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/capella2polarion/elements/serialize.py b/capella2polarion/elements/serialize.py index 1f192799..7a1ee35d 100644 --- a/capella2polarion/elements/serialize.py +++ b/capella2polarion/elements/serialize.py @@ -138,6 +138,9 @@ def _get_requirement_types_text( continue if not (req.type and req.text): + logger.warning( + "Requirement without text or type found %r", req.name + ) continue type_texts[req.type.long_name].append(req.text) diff --git a/tests/test_elements.py b/tests/test_elements.py index de07e915..7d0f5800 100644 --- a/tests/test_elements.py +++ b/tests/test_elements.py @@ -62,6 +62,12 @@ TEST_WI_CHECKSUM = ( "73508ec0c3048c5b33316dfa56ef5e5f4179ff69efaa209e47ab65b111415e82" ) +TEST_REQ_TEXT = ( + "

    Test requirement 1 really l o n g text that is way too long to " + "display here as that

    \n\n

    < > " '

    \n\n
      \n\t
    • " + "This is a list
      • \n\t
    • an unordered one
      • \n
    \n\n
      \n\t" + "
    1. Ordered list
      2. \n\t
    2. Ok
      3. \n
    \n" +) class TestDiagramElements: @@ -478,7 +484,7 @@ def test__decode_diagram(): @pytest.mark.parametrize( "uuid,expected", [ - ( + pytest.param( TEST_ELEMENT_UUID, { "type": "logicalComponent", @@ -486,9 +492,14 @@ def test__decode_diagram(): "uuid_capella": TEST_ELEMENT_UUID, "description_type": "text/html", "description": markupsafe.Markup(TEST_DESCR), + "reqtype": { + "type": "text/html", + "value": markupsafe.Markup(TEST_REQ_TEXT), + }, }, + id="logicalComponent", ), - ( + pytest.param( TEST_OCAP_UUID, { "type": "operationalCapability", @@ -501,8 +512,9 @@ def test__decode_diagram(): "postCondition": {"type": "text/html", "value": ""}, }, }, + id="operationalCapability", ), - ( + pytest.param( TEST_WE_UUID, { "type": "entity", @@ -511,8 +523,9 @@ def test__decode_diagram(): "description_type": "text/html", "description": markupsafe.Markup(TEST_WE_DESCR), }, + id="entity", ), - ( + pytest.param( TEST_ACTOR_UUID, { "type": "logicalActor", @@ -524,8 +537,9 @@ def test__decode_diagram(): "and greatest mage of all time.

    \n" ), }, + id="logicalActor", ), - ( + pytest.param( TEST_PHYS_COMP, { "type": "physicalComponent", @@ -534,8 +548,9 @@ def test__decode_diagram(): "description_type": "text/html", "description": markupsafe.Markup(""), }, + id="physicalComponent", ), - ( + pytest.param( TEST_PHYS_NODE, { "type": "physicalComponentNode", @@ -544,8 +559,9 @@ def test__decode_diagram(): "description_type": "text/html", "description": markupsafe.Markup(""), }, + id="physicalComponentNode", ), - ( + pytest.param( TEST_SCENARIO, { "type": "scenario", @@ -558,8 +574,9 @@ def test__decode_diagram(): "postCondition": {"type": "text/html", "value": ""}, }, }, + id="scenario", ), - ( + pytest.param( TEST_CAP_REAL, { "type": "capabilityRealization", @@ -572,8 +589,9 @@ def test__decode_diagram(): "postCondition": {"type": "text/html", "value": ""}, }, }, + id="capabilityRealization", ), - ( + pytest.param( TEST_CONSTRAINT, { "type": "constraint", @@ -584,6 +602,7 @@ def test__decode_diagram(): "This is a test context.Make Food" ), }, + id="constraint", ), ], )