From f72e7036afbb5649db592fab264d6d00cc544b49 Mon Sep 17 00:00:00 2001 From: Plamen Dimitrov Date: Tue, 16 Jul 2024 03:33:03 +0700 Subject: [PATCH] Provide Pyro5 compatibility to the remote door utility The main adaptation that was needed comes down to dropping the previous expose-free approach and dynamically exposing each relevant function and class in addition to the previous autoproxy steps. To allow for modules and classes that the shared remote object has not explicitly imported or are not detectable as needing exposing, the previous whitelist argument has been extended to allow exposing predefined classes and serializing predefined exceptions that could be used and raised during remote use. Signed-off-by: Plamen Dimitrov --- aexpect/remote_door.py | 141 +++++++++++++++++++++++++++++++++----- tests/test_remote_door.py | 52 ++++++++++++-- 2 files changed, 172 insertions(+), 21 deletions(-) diff --git a/aexpect/remote_door.py b/aexpect/remote_door.py index 36b4a13..8a539ba 100644 --- a/aexpect/remote_door.py +++ b/aexpect/remote_door.py @@ -70,8 +70,25 @@ # related to connectivity and perform further development on this utility. # os.environ["PYRO_LOGLEVEL"] = "DEBUG" try: - # noinspection PyPackageRequirements,PyUnresolvedReferences - import Pyro4 + try: + # noinspection PyPackageRequirements,PyUnresolvedReferences + from Pyro5.compatibility import Pyro4 + # noinspection PyPackageRequirements + from Pyro5 import server + # noinspection PyPackageRequirements + from Pyro5 import nameserver + except ImportError: + # noinspection PyPackageRequirements,PyUnresolvedReferences + import Pyro4 + # Pyro5 compatibility does not support submodules so we need separate handling + from unittest.mock import Mock + server = Mock() + server.expose = Pyro4.expose + server.is_private_attribute = Pyro4.util.is_private_attribute + # noinspection PyPackageRequirements + from Pyro4 import naming as nameserver + nameserver.start_ns = nameserver.startNS + except ImportError: logging.warning("Remote object backend (Pyro4) not found, some functionality" " of the remote door will not be available") @@ -438,7 +455,7 @@ def set_subcontrol_parameter_object(subcontrol, value): LOG.info("Sharing the test parameters over the network") Pyro4.config.AUTOPROXY = False - Pyro4.config.REQUIRE_EXPOSE = False + expose_remote_classes(["virttest.utils_params.Params"]) try: pyro_daemon = Pyro4.Daemon(host=host_ip, port=1437) LOG.debug("Pyro4 daemon started successfully") @@ -496,7 +513,8 @@ def run(self): self.pyro_daemon.requestLoop() -def get_remote_object(object_name, session=None, host="localhost", port=9090): +def get_remote_object(object_name, session=None, host="localhost", port=9090, + object_wl=None, expose_wl=None, serialize_wl=None): """ Get a data object (visual or other) executing remotely or share one if none is available, generating control files along @@ -508,6 +526,16 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): :type session: RemoteSession or None :param str host: ip address of the local sharing server :param int port: port of the local name server + :param object_wl: whitelist of allowed functions/classes of the local (typically + module) object as a tuple pair (functions, classes); + expose and autoproxy all (do not filter) if None or empty + :type object_wl: ([str],[str]) or None + :param expose_wl: extra classes to expose as a pair of full module class paths + and general module paths to expose all classes from + :type expose_wl: ([str],[str]) or None + :param serialize_wl: exceptions to serialize as as a pair of full module class paths + and general module paths to serialize all exceptions from + :type serialize_wl: ([str],[str]) or None :returns: proxy version of the remote object :rtype: Pyro4.Proxy @@ -543,7 +571,11 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): # if there is no door on the other side, open one _copy_control(session, os.path.abspath(__file__), is_utility=True) run_remote_util(session, "remote_door", "share_local_object", - object_name, host=host, port=port, detach=True) + object_name, host=host, port=port, + object_wl=object_wl, + expose_wl=expose_wl, + serialize_wl=serialize_wl, + detach=True) output, attempts = "", 10 for _ in range(attempts): output = session.get_output() @@ -611,26 +643,35 @@ def get_remote_objects(session=None, host="localhost", port=0): return remote_objects -def share_local_object(object_name, whitelist=None, host="localhost", port=9090): +def share_local_object(object_name, host="localhost", port=9090, + object_wl=None, expose_wl=None, serialize_wl=None): """ Share a local object of the given name over the network. :param str object_name: name of the local object - :param whitelist: shared functions/classes of the local object as tuple - pairs (module, function) or (module, class); whitelist - all (do not filter) if None or empty - :type whitelist: [(str,str)] or None :param str host: ip address of the local name server :param port: port of the local sharing server :type port: int or str + :param object_wl: whitelist of allowed functions/classes of the local (typically + module) object as a tuple pair (functions, classes); + expose and autoproxy all (do not filter) if None or empty + :type object_wl: ([str],[str]) or None + :param expose_wl: extra classes to expose as a pair of full module class paths + and general module paths to expose all classes from + :type expose_wl: ([str],[str]) or None + :param serialize_wl: exceptions to serialize as as a pair of full module class paths + and general module paths to serialize all exceptions from + :type serialize_wl: ([str],[str]) or None This function shares a custom object with whitelisted attributes through a custom implementation. It is more secure but more limited as functionality since it requires serialization extensions. """ Pyro4.config.AUTOPROXY = True - Pyro4.config.REQUIRE_EXPOSE = False port = int(port) if isinstance(port, str) else port + object_wl = object_wl or ([], []) + expose_wl = expose_wl or ([], []) + serialize_wl = serialize_wl or ([], []) # pyro daemon try: @@ -653,9 +694,7 @@ def share_local_object(object_name, whitelist=None, host="localhost", port=9090) LOG.debug("Pyro4 name server already started") # network unreachable and failed to locate the nameserver error except (OSError, Pyro4.errors.NamingError): - # noinspection PyPackageRequirements - from Pyro4 import naming - ns_uri, ns_daemon, _bc_server = naming.startNS(host=host, port=port) + ns_uri, ns_daemon, _bc_server = nameserver.start_ns(host=host, port=port) ns_server = Pyro4.Proxy(ns_uri) LOG.debug("Pyro4 name server started successfully with URI %s", ns_uri) @@ -696,12 +735,31 @@ class ModuleObject: # pylint: disable=R0903 """Module wrapped for transferability.""" for fname, fobj in inspect.getmembers(module, inspect.isfunction): - if not whitelist or (object_name, fname) in whitelist: + if not object_wl[0] or fname in object_wl[0]: + LOG.info("Autoproxying %s's function %s", object_name, fname) setattr(ModuleObject, fname, staticmethod(proxymethod(fobj))) + if not server.is_private_attribute(fname): + LOG.info("Exposing %s's function %s", object_name, fname) + try: + server.expose(fobj) + except AttributeError as error: + LOG.warning(error) for cname, cobj in inspect.getmembers(module, inspect.isclass): - if not whitelist or (object_name, cname) in whitelist: + if not object_wl[1] or cname in object_wl[1]: + LOG.info("Autoproxying %s's function %s", object_name, cname) setattr(ModuleObject, cname, staticmethod(proxymethod(cobj))) + if not server.is_private_attribute(cname): + LOG.info("Exposing %s's function %s", object_name, cname) + try: + server.expose(cobj) + except AttributeError as error: + LOG.warning(error) + local_object = ModuleObject() + server.expose(ModuleObject) + # additional classes to expose and exceptions to serialize + expose_remote_classes(*expose_wl) + import_remote_exceptions(*serialize_wl) # we should register to the pyro daemon before entering its loop uri = pyro_daemon.register(local_object) @@ -838,7 +896,7 @@ def import_remote_exceptions(exceptions=None, modules=None): :type exceptions: [str] or None :param modules: full module paths whose custom exceptions will first be detected and then automatically imported (optional) - :type exceptions: [str] or None + :type modules: [str] or None The deserialization by our Pyro backend requires the full module paths to each exception or module in order to correctly detect the exception type. @@ -885,3 +943,52 @@ def recreate_exception(class_name, class_dict): for exception in exceptions: # noinspection PyUnresolvedReferences Pyro4.util.SerializerBase.register_dict_to_class(exception, recreate_exception) + + +def expose_remote_classes(classes=None, modules=None): + """ + Make accessible all remote custom classes. + + :param classes: full module path class names (optional) + :type classes: [str] or None + :param modules: full module paths whose custom classes will first be + detected and then automatically exposed inclusive of + parent classes and inheritance (optional) + :type modules: [str] or None + """ + def list_module_classes(modstr): + imported_module = importlib.import_module(modstr) + module_classes = [] + for name in imported_module.__dict__: + if not inspect.isclass(imported_module.__dict__[name]): + continue + module_classes.append(modstr + "." + name) + return module_classes + + classes = [] if not classes else classes + modules = [] if not modules else modules + for module in modules: + classes += list_module_classes(module) + LOG.debug("Exposing the following classes (with proper inheritance): %s", + ", ".join(classes)) + + def get_class_from_name(clsstr): + module_name, class_name = clsstr.rsplit('.', 1) + module = importlib.import_module(module_name) + cls = getattr(module, class_name) + return cls + + for cls in map(get_class_from_name, classes): + # the inheritance is inclusive of the class itself + for base_cls in cls.__mro__: + if base_cls in (object, int, float, bool, str, tuple, frozenset): + # known immutable classes should be skipped + continue + if inspect.ismethoddescriptor(base_cls): + # method descriptors should be skipped + continue + try: + server.expose(base_cls) + except (TypeError, AttributeError) as error: + logging.warning("Additional class exposing error: %s", error) + continue diff --git a/tests/test_remote_door.py b/tests/test_remote_door.py index 7d93520..afc4b2f 100644 --- a/tests/test_remote_door.py +++ b/tests/test_remote_door.py @@ -19,8 +19,9 @@ import re import shutil import unittest.mock +import html -from aexpect import remote_door +from aexpect import client, remote, remote_door from aexpect.client import RemoteSession mock = unittest.mock @@ -153,6 +154,8 @@ def test_get_remote_object(self): self.session = mock.MagicMock(name='session') self.session.client = "ssh" remote_door.Pyro4 = mock.MagicMock() + remote_door.server = mock.MagicMock() + remote_door.nameserver = mock.MagicMock() disconnect = remote_door.Pyro4.errors.PyroError = Exception remote_door.Pyro4.Proxy.side_effect = [disconnect("no such object"), mock.DEFAULT] self.session.get_output.return_value = "Local object sharing ready\n" @@ -180,11 +183,19 @@ def test_get_remote_object(self): control_lines = handle.readlines() self.assertIn("import remote_door\n", control_lines) self.assertIn("result = remote_door.share_local_object(r'html', " - "host=r'testhost', port=4242)\n", + "expose_wl=None, host=r'testhost', object_wl=None, " + "port=4242, serialize_wl=None)\n", control_lines) - # since the local run was face redo it here - remote_door.share_local_object("html", None, "testhost", 4242) + # since the local run was fake redo it here + remote_door.server.is_private_attribute = lambda x: False + remote_door.share_local_object("html", "testhost", 4242) + remote_door.server.expose.assert_called() + # TODO: to make the remote door usable with nearly no dependencies + # we use a lot of internal classes and functions to make sharing a + # local object possible but this makes testing these less accessible + remote_door.server.expose.assert_any_call(html.escape) + remote_door.server.expose.assert_any_call(html.unescape) def test_share_remote_objects(self): """Test that a remote object can be shared properly and remotely.""" @@ -239,3 +250,36 @@ def get_first_arg(call): # assert some detected exceptions from the remote module self.assertIn("aexpect.remote.RemoteError", imported_classes) self.assertIn("aexpect.remote.UDPError", imported_classes) + + def test_expose_remote_classes(self): + """Test that selected remote classes are properly exposed.""" + remote_door.Pyro4 = mock.MagicMock() + remote_door.server = mock.MagicMock() + preselected_classes = ["aexpect.client.ShellSession", + "aexpect.remote.LoginError"] + remote_door.expose_remote_classes(preselected_classes) + expose_method = remote_door.server.expose + self.assertEqual(expose_method.mock_calls[0], + mock.call(client.ShellSession)) + self.assertEqual(expose_method.mock_calls[1], + mock.call(client.Expect)) + self.assertEqual(expose_method.mock_calls[2], + mock.call(client.Tail)) + self.assertEqual(expose_method.mock_calls[3], + mock.call(client.Spawn)) + self.assertNotEqual(expose_method.mock_calls[4], + mock.call(object)) + self.assertEqual(expose_method.mock_calls[4], + mock.call(remote.LoginError)) + self.assertEqual(expose_method.mock_calls[5], + mock.call(remote.RemoteError)) + + expose_method.reset_mock() + preselected_modules = ["aexpect.client"] + remote_door.expose_remote_classes([], modules=preselected_modules) + self.assertIn(mock.call(client.ExpectError), + expose_method.mock_calls, + "classes imported from elsewhere are exposed") + self.assertIn(mock.call(client.ShellSession), + expose_method.mock_calls, + "defined classes are exposed")