Fix/nsmap copy bug (Draegerwerk#245)

<!--- Provide a general summary of your changes in the title above -->
<!--- Link the corresponding issues after you created the pull request
-->

## Types of changes
<!--- What types of changes does your code introduce? Put an `x` in all
the boxes that apply: -->
- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)

## Checklist:
<!--- Go over all the following points, and put an `x` in all the boxes
that apply. -->
<!--- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
- [x] I have updated the [changelog](../CHANGELOG.md) accordingly.
- [x] I have added tests to cover my changes.

CHANGELOG.md

@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - fixed a bug where the `SdcConsumer` failed to determine the host network adapter if the ip contained in the `device_location` is on a different subnet
 - comparison of extensions would fail [#238](https://github.com/Draegerwerk/sdc11073/issues/238)
 - ExtensionLocalValue.value must be a list instead of a dictionary in order to allow multiple elements with same name.
+- fixed a bug where namespaces of xml are lost when coping lxml elements
 
 
 ### Changed

examples/requestmanipulator.py

@@ -2,7 +2,7 @@
 from typing import TYPE_CHECKING, Union
 
 if TYPE_CHECKING:
-    from lxml.etree import ElementBase
+    from sdc11073 import xml_utils
     from sdc11073.pysoap.soapenvelope import Soap12Envelope
 
 
@@ -26,7 +26,7 @@ def manipulate_soapenvelope(self, soap_envelope: Soap12Envelope) -> Union[Soap12
             return self.cb_soapenvelope(soap_envelope)
         return None
 
-    def manipulate_domtree(self, domtree: ElementBase) -> Union[ElementBase, None]:
+    def manipulate_domtree(self, domtree: xml_utils.LxmlElement) -> Union[xml_utils.LxmlElement, None]:
         if callable(self.cb_xml):
             return self.cb_xml(domtree)
         return None

pyproject.toml

@@ -172,7 +172,7 @@ max-doc-length = 120 # https://beta.ruff.rs/docs/settings/#max-doc-length
 "__init__.py" = ["D104"]
 
 [tool.mypy]
-python_version = 3.9 # https://mypy.readthedocs.io/en/stable/config_file.html#confval-python_version
+python_version = "3.9" # https://mypy.readthedocs.io/en/stable/config_file.html#confval-python_version
 strict = true # https://mypy.readthedocs.io/en/stable/config_file.html#confval-strict
 disallow_untyped_calls = true # https://mypy.readthedocs.io/en/stable/config_file.html#confval-disallow_untyped_calls
 disallow_untyped_defs = true # https://mypy.readthedocs.io/en/stable/config_file.html#confval-disallow_untyped_defs

src/sdc11073/consumer/consumerimpl.py

@@ -3,7 +3,6 @@
 
 import copy
 import ssl
-import time
 import traceback
 import uuid
 from dataclasses import dataclass
@@ -13,7 +12,7 @@
 from lxml import etree as etree_
 
 import sdc11073.certloader
-from sdc11073 import commlog, loghelper, network
+from sdc11073 import commlog, loghelper, network, xml_utils
 from sdc11073 import observableproperties as properties
 from sdc11073.definitions_base import ProtocolsRegistry
 from sdc11073.dispatch import DispatchKey, MessageConverterMiddleware
@@ -30,7 +29,6 @@
 from .request_handler_deferred import EmptyResponse
 
 if TYPE_CHECKING:
-    from ssl import SSLContext
     from collections.abc import Iterable
     from sdc11073.dispatch.request import RequestData
     from sdc11073.xml_types.mex_types import HostedServiceType
@@ -41,7 +39,6 @@
     from sdc11073.mdib.consumermdib import ConsumerMdib
     from sdc11073.consumer.serviceclients.serviceclientbase import HostedServiceClient
     from sdc11073.consumer.subscription import ConsumerSubscriptionManagerProtocol
-    from sdc11073.loghelper import LoggerAdapter
     from sdc11073.wsdiscovery.service import Service
     from .components import SdcConsumerComponents
 
@@ -65,7 +62,7 @@ def __init__(self, service_id: str,  # noqa: PLR0913
         self.log_prefix = log_prefix
         self.meta_data: mex_types.Metadata | None = None
         self.wsdl_string = None
-        self.wsdl_node: etree_.ElementBase | None = None
+        self.wsdl_node: xml_utils.LxmlElement | None = None
         self._logger = loghelper.get_logger_adapter('sdc.client.hosted', log_prefix)
         self._url = urlparse(endpoint_address)
         self.services = {}
@@ -341,7 +338,7 @@ def do_subscribe(self, dpws_hosted: HostedServiceType,  # noqa: PLR0913
                      filter_type: eventing_types.FilterType,
                      actions: Iterable[DispatchKey],
                      expire_minutes: int = 60,
-                     any_elements: list | None = None,
+                     any_elements: list[xml_utils.LxmlElement] | None = None,
                      any_attributes: dict | None = None) -> ConsumerSubscription:
         """Send subscribe request to provider.
 
@@ -350,7 +347,7 @@ def do_subscribe(self, dpws_hosted: HostedServiceType,  # noqa: PLR0913
         :param filter_type: the filter that is sent to device
         :param actions: a list of DispatchKeys that this subscription shall handle
         :param expire_minutes: defaults to 1 hour
-        :param any_elements: optional list of etree.ElementBase objects
+        :param any_elements: optional list of lxml elements
         :param any_attributes: optional dictionary of name:str - value:str pairs
         :return: a subscription object that has callback already registered.
         """

src/sdc11073/consumer/manipulator.py

@@ -3,7 +3,7 @@
 from typing import TYPE_CHECKING, Protocol
 
 if TYPE_CHECKING:
-    from lxml.etree import ElementBase
+    from sdc11073 import xml_utils
 
     from sdc11073.pysoap.soapenvelope import Soap12Envelope
 
@@ -20,7 +20,7 @@ class RequestManipulatorProtocol(Protocol):
     def manipulate_soapenvelope(self, soap_envelope: Soap12Envelope) -> Soap12Envelope | None:
         """Manipulate on Soap12Envelope level."""
 
-    def manipulate_domtree(self, domtree: ElementBase) -> ElementBase | None:
+    def manipulate_domtree(self, domtree: xml_utils.LxmlElement) -> xml_utils.LxmlElement | None:
         """Manipulate on etree.Element level."""
 
     def manipulate_string(self, xml_string: str) -> str | None:

src/sdc11073/consumer/subscription.py

@@ -28,6 +28,7 @@
     from sdc11073.pysoap.soapclient import SoapClientProtocol
     from sdc11073.xml_types.eventing_types import FilterType
     from sdc11073.xml_types.mex_types import HostedServiceType
+    from sdc11073 import xml_utils
 
 
 class ConsumerSubscriptionProtocol(Protocol):
@@ -90,9 +91,9 @@ def __init__(self, msg_factory: MessageFactory,  # noqa: PLR0913
         self.requested_expires: int | float = 0
         self.granted_expires: int | float = 0
 
-        self.notify_to_identifier: etree_.ElementBase | None = None
+        self.notify_to_identifier: xml_utils.LxmlElement | None = None
 
-        self.end_to_identifier: etree_.ElementBase | None = None
+        self.end_to_identifier: xml_utils.LxmlElement | None = None
 
         self._logger = loghelper.get_logger_adapter('sdc.client.subscr', log_prefix)
         self.event_counter = 0  # for display purpose, we count notifications

src/sdc11073/mdib/containerbase.py

@@ -4,10 +4,11 @@
 import inspect
 from typing import Any
 
-from lxml.etree import Element, ElementBase, QName
+from lxml.etree import Element, QName
 
 from sdc11073 import observableproperties as properties
 from sdc11073.namespaces import QN_TYPE, NamespaceHelper
+from sdc11073 import xml_utils
 
 
 class ContainerBase:
@@ -33,7 +34,7 @@ def get_actual_value(self, attr_name: str) -> Any:
         """Ignore default value and implied value, e.g. return None if value is not present in xml."""
         return getattr(self.__class__, attr_name).get_actual_value(self)
 
-    def mk_node(self, tag: QName, ns_helper: NamespaceHelper, set_xsi_type: bool = False) -> ElementBase:
+    def mk_node(self, tag: QName, ns_helper: NamespaceHelper, set_xsi_type: bool = False) -> xml_utils.LxmlElement:
         """Create an etree node from instance data.
 
         :param tag: tag of the newly created node
@@ -48,9 +49,9 @@ def mk_node(self, tag: QName, ns_helper: NamespaceHelper, set_xsi_type: bool = F
         self.update_node(node, ns_helper, set_xsi_type)
         return node
 
-    def update_node(self, node: ElementBase,
+    def update_node(self, node: xml_utils.LxmlElement,
                     ns_helper: NamespaceHelper,
-                    set_xsi_type: bool = False) -> ElementBase:
+                    set_xsi_type: bool = False) -> xml_utils.LxmlElement:
         """Update node with own data.
 
         :param node: node to be updated
@@ -64,7 +65,7 @@ def update_node(self, node: ElementBase,
             prop.update_xml_value(self, node)
         return node
 
-    def update_from_node(self, node: ElementBase):
+    def update_from_node(self, node: xml_utils.LxmlElement):
         """Update members from node."""
         for _, cprop in self.sorted_container_properties():
             cprop.update_from_node(self, node)
@@ -82,8 +83,8 @@ def _update_from_other(self, other_container: ContainerBase, skipped_properties:
     def mk_copy(self, copy_node: bool = True) -> ContainerBase:
         """Make a copy of self."""
         copied = copy.copy(self)
-        if copy_node:
-            copied.node = copy.deepcopy(self.node)
+        if copy_node and self.node is not None:
+            copied.node = xml_utils.copy_element(self.node)
         return copied
 
     def sorted_container_properties(self) -> list:

src/sdc11073/mdib/descriptorcontainers.py

@@ -18,6 +18,7 @@
     from decimal import Decimal
 
     from lxml import etree as etree_
+    from sdc11073 import xml_utils
 
     from sdc11073.namespaces import NamespaceHelper
     from sdc11073.xml_types.isoduration import DurationType
@@ -86,7 +87,7 @@ def set_source_mds(self, handle: str):
         ...
 
     @classmethod
-    def from_node(cls, node: etree_.ElementBase, parent_handle: str | None = None) -> AbstractDescriptorProtocol:
+    def from_node(cls, node: xml_utils.LxmlElement, parent_handle: str | None = None) -> AbstractDescriptorProtocol:
         """Create class and init its properties from the node."""
 
 
@@ -201,7 +202,7 @@ def diff(self, other: AbstractDescriptorContainer, ignore_property_names: list[s
         return None if len(ret) == 0 else ret
 
     def mk_descriptor_node(self, tag: etree_.QName,
-                           ns_helper: NamespaceHelper, set_xsi_type: bool = True) -> etree_.ElementBase:
+                           ns_helper: NamespaceHelper, set_xsi_type: bool = True) -> xml_utils.LxmlElement:
         """Create a lxml etree node from instance data.
 
         :param tag: tag of node
@@ -226,7 +227,7 @@ def tag_name_for_child_descriptor(self, node_type: etree_.QName) -> (etree_.QNam
                 return child.child_qname, set_xsi_type
         raise ValueError(f'{node_type} not known in child declarations of {self.__class__.__name__}')
 
-    def sort_child_nodes(self, node: etree_.ElementBase) -> None:
+    def sort_child_nodes(self, node: xml_utils.LxmlElement) -> None:
         """Bring all child elements of node in correct order (BICEPS schema).
 
         raises a ValueError if a child node exist that is not listed in ordered_tags
@@ -265,7 +266,7 @@ def __repr__(self) -> str:
                 f'parent={self.parent_handle}')
 
     @classmethod
-    def from_node(cls, node: etree_.ElementBase, parent_handle: str | None = None) -> AbstractDescriptorContainer:
+    def from_node(cls, node: xml_utils.LxmlElement, parent_handle: str | None = None) -> AbstractDescriptorContainer:
         """Create class and init its properties from the node."""
         obj = cls(handle=None,  # will be determined in constructor from node value
                   parent_handle=parent_handle)

src/sdc11073/mdib/mdibbase.py

@@ -17,6 +17,7 @@
     from sdc11073.definitions_base import BaseDefinitions
     from sdc11073.loghelper import LoggerAdapter
     from sdc11073.xml_types.pm_types import CodedValue
+    from sdc11073 import xml_utils
 
     from .descriptorcontainers import AbstractDescriptorContainer, AbstractOperationDescriptorContainer
     from .statecontainers import AbstractMultiStateContainer, AbstractStateContainer
@@ -348,7 +349,7 @@ def add_state_containers(self, state_containers: list[AbstractStateContainer | A
                     self._logger.error('add_state_containers: {}, DescriptorHandle={}; {}', # noqa: PLE1205
                                        ex, state_container.DescriptorHandle, traceback.format_exc())
 
-    def _reconstruct_md_description(self) -> etree_.ElementBase:
+    def _reconstruct_md_description(self) -> xml_utils.LxmlElement:
         """Build dom tree of descriptors from current data."""
         pm = self.data_model.pm_names
         doc_nsmap = self.nsmapper.ns_map
@@ -364,7 +365,7 @@ def _reconstruct_md_description(self) -> etree_.ElementBase:
     def make_descriptor_node(self,
                              descriptor_container: AbstractDescriptorContainer,
                              tag: etree_.QName,
-                             set_xsi_type: bool = True) -> etree_.ElementBase:
+                             set_xsi_type: bool = True) -> xml_utils.LxmlElement:
         """Create a lxml etree node with subtree from instance data.
 
         :param descriptor_container: a descriptor container instance
@@ -387,7 +388,7 @@ def make_descriptor_node(self,
         descriptor_container.sort_child_nodes(node)
         return node
 
-    def _reconstruct_mdib(self, add_context_states: bool) -> etree_.ElementBase:
+    def _reconstruct_mdib(self, add_context_states: bool) -> xml_utils.LxmlElement:
         """Build dom tree of mdib from current data.
 
         If add_context_states is False, context states are not included.
@@ -413,21 +414,21 @@ def _reconstruct_mdib(self, add_context_states: bool) -> etree_.ElementBase:
                 md_state_node.append(state_container.mk_state_node(tag, self.nsmapper))
         return mdib_node
 
-    def reconstruct_md_description(self) -> (etree_.ElementBase, MdibVersionGroup):
+    def reconstruct_md_description(self) -> (xml_utils.LxmlElement, MdibVersionGroup):
         """Build dom tree of descriptors from current data."""
         with self.mdib_lock:
             node = self._reconstruct_md_description()
             return node, self.mdib_version_group
 
-    def reconstruct_mdib(self) -> (etree_.ElementBase, MdibVersionGroup):
+    def reconstruct_mdib(self) -> (xml_utils.LxmlElement, MdibVersionGroup):
         """Build dom tree from current data.
 
         This method does not include context states!
         """
         with self.mdib_lock:
             return self._reconstruct_mdib(add_context_states=False), self.mdib_version_group
 
-    def reconstruct_mdib_with_context_states(self) -> (etree_.ElementBase, MdibVersionGroup):
+    def reconstruct_mdib_with_context_states(self) -> (xml_utils.LxmlElement, MdibVersionGroup):
         """Build dom tree from current data.
 
         This method includes the context states.

src/sdc11073/mdib/statecontainers.py

@@ -16,12 +16,13 @@
 if TYPE_CHECKING:
     from decimal import Decimal
 
-    from lxml.etree import ElementBase, QName
+    from lxml.etree import QName
 
     from sdc11073.location import SdcLocation
     from sdc11073.namespaces import NamespaceHelper
     from sdc11073.xml_types.isoduration import DurationType
     from sdc11073.xml_types.xml_structure import ExtensionLocalValue
+    from sdc11073 import xml_utils
 
     from .descriptorcontainers import AbstractDescriptorProtocol
 
@@ -52,7 +53,7 @@ def update_from_other_container(self, other: AbstractStateProtocol,
                                     skipped_properties: list[str] | None = None):
         """Copy all properties except the skipped ones to self."""
 
-    def update_from_node(self, node: ElementBase):
+    def update_from_node(self, node: xml_utils.LxmlElement):
         """Update members from node."""
 
 
@@ -88,7 +89,7 @@ def __init__(self, descriptor_container: AbstractDescriptorProtocol):
 
     def mk_state_node(self, tag: QName,
                       nsmapper: NamespaceHelper,
-                      set_xsi_type: bool = True) -> ElementBase:
+                      set_xsi_type: bool = True) -> xml_utils.LxmlElement:
         """Create an etree node from instance data."""
         return super().mk_node(tag, nsmapper, set_xsi_type=set_xsi_type)
 
@@ -122,7 +123,7 @@ def __repr__(self) -> str:
         return f'{self.__class__.__name__} DescriptorHandle="{self.DescriptorHandle}" StateVersion={self.StateVersion}'
 
     @classmethod
-    def from_node(cls, node: ElementBase,
+    def from_node(cls, node: xml_utils.LxmlElement,
                   descriptor_container: AbstractDescriptorProtocol | None = None) -> AbstractStateContainer:
         """Create an instance from XML node."""
         obj = cls(descriptor_container)
@@ -553,7 +554,7 @@ def update_from_other_container(self, other: AbstractMultiStateContainer, skippe
                 f'Update from a node with different handle is not possible! Have "{self.Handle}", got "{other.Handle}"')
         super().update_from_other_container(other, skipped_properties)
 
-    def mk_state_node(self, tag: QName, nsmapper: NamespaceHelper, set_xsi_type: bool = True) -> ElementBase:
+    def mk_state_node(self, tag: QName, nsmapper: NamespaceHelper, set_xsi_type: bool = True) -> xml_utils.LxmlElement:
         """Create an etree node from instance data."""
         if self.Handle is None:
             self.Handle = uuid.uuid4().hex

src/sdc11073/provider/dpwshostedservice.py

@@ -10,6 +10,7 @@
 from ..xml_types import mex_types
 from ..xml_types.addressing_types import EndpointReferenceType
 from ..xml_types.dpws_types import HostedServiceType
+from sdc11073 import xml_utils
 
 _wsdl_ns = ns_hlp.WSDL.namespace
 
@@ -22,7 +23,7 @@
 WSDL_S12 = ns_hlp.WSDL12.namespace  # old soap 12 namespace, used in wsdl 1.1. used only for wsdl
 
 
-def etree_from_file(path) -> etree_.ElementBase:
+def etree_from_file(path) -> xml_utils.LxmlElement:
     parser = etree_.ETCompatXMLParser(resolve_entities=False)
     doc = etree_.parse(path, parser=parser)
     return doc.getroot()

src/sdc11073/provider/porttypes/descriptioneventserviceimpl.py

@@ -9,7 +9,7 @@
 if TYPE_CHECKING:
     from ...mdib.descriptorcontainers import AbstractDescriptorContainer
     from ...mdib.statecontainers import AbstractStateContainer
-    from lxml import etree as etree_
+    from sdc11073 import xml_utils
 
 class DescriptionEventService(DPWSPortTypeBase):
     port_type_name = PrefixesEnum.SDC.tag('DescriptionEventService')
@@ -39,7 +39,7 @@ def send_descriptor_updates(self, updated: List[AbstractDescriptorContainer],
         subscription_mgr.send_to_subscribers(body_node, action, mdib_version_group, 'send_descriptor_updates')
 
     def mk_description_modification_report_body(self, mdib_version_group, updated, created, deleted,
-                                                updated_states) -> etree_.ElementBase:
+                                                updated_states) -> xml_utils.LxmlElement:
         # This method creates one ReportPart for every descriptor.
         # An optimization is possible by grouping all descriptors with the same parent handle into one ReportPart.
         # This is not implemented, and I think it is not needed.