Provide Pyro5 compatibility to the remote door utility

The main adaptation that was needed comes down to dropping the
previous expose-free approach and dynamically exposing each
relevant function and class in addition to the previous autoproxy
steps.

To allow for modules and classes that the shared remote object has
not explicitly imported or are not detectable as needing exposing,
the previous whitelist argument has been extended to allow exposing
predefined classes and serializing predefined exceptions that could
be used and raised during remote use.

Signed-off-by: Plamen Dimitrov <[email protected]>

aexpect/remote_door.py

@@ -70,8 +70,27 @@
 # related to connectivity and perform further development on this utility.
 # os.environ["PYRO_LOGLEVEL"] = "DEBUG"
 try:
-    # noinspection PyPackageRequirements,PyUnresolvedReferences
-    import Pyro4
+    try:
+        # noinspection PyPackageRequirements,PyUnresolvedReferences
+        from Pyro5.compatibility import Pyro4
+        # noinspection PyPackageRequirements
+        from Pyro5 import server
+        # noinspection PyPackageRequirements
+        from Pyro5 import nameserver
+    except ImportError:
+        # noinspection PyPackageRequirements,PyUnresolvedReferences
+        import Pyro4
+        # Pyro5 compatibility does not support submodules so we need separate handling
+        class Server:
+            def __init__(self):
+                self.__dict__ = {}
+        server = Server()
+        server.expose = Pyro4.expose
+        server.is_private_attribute = Pyro4.util.is_private_attribute
+        # noinspection PyPackageRequirements
+        from Pyro4 import naming as nameserver
+        nameserver.start_ns = nameserver.startNS
+
 except ImportError:
     logging.warning("Remote object backend (Pyro4) not found, some functionality"
                     " of the remote door will not be available")
@@ -411,7 +430,7 @@ def set_subcontrol_parameter_dict(subcontrol, dict_name, value):
 
 
 @set_subcontrol
-def set_subcontrol_parameter_object(subcontrol, value):
+def set_subcontrol_parameter_object(subcontrol, object_type, value):
     """
     Prepare a URI to remote params for the remote control file.
 
@@ -438,7 +457,7 @@ def set_subcontrol_parameter_object(subcontrol, value):
 
     LOG.info("Sharing the test parameters over the network")
     Pyro4.config.AUTOPROXY = False
-    Pyro4.config.REQUIRE_EXPOSE = False
+    expose_remote_classes([object_type])
     try:
         pyro_daemon = Pyro4.Daemon(host=host_ip, port=1437)
         LOG.debug("Pyro4 daemon started successfully")
@@ -496,7 +515,8 @@ def run(self):
         self.pyro_daemon.requestLoop()
 
 
-def get_remote_object(object_name, session=None, host="localhost", port=9090):
+def get_remote_object(object_name, session=None, host="localhost", port=9090,
+                      object_wl=None, expose_wl=None, serialize_wl=None):
     """
     Get a data object (visual or other) executing remotely or
     share one if none is available, generating control files along
@@ -508,6 +528,16 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090):
     :type session: RemoteSession or None
     :param str host: ip address of the local sharing server
     :param int port: port of the local name server
+    :param object_wl: whitelist of allowed functions/classes of the local (typically
+                      module) object as a tuple pair (functions, classes);
+                      expose and autoproxy all (do not filter) if None or empty
+    :type object_wl: ([str],[str]) or None
+    :param expose_wl: extra classes to expose as a pair of full module class paths
+                      and general module paths to expose all classes from
+    :type expose_wl: ([str],[str]) or None
+    :param serialize_wl: exceptions to serialize as a pair of full module class paths
+                         and general module paths to serialize all exceptions from
+    :type serialize_wl: ([str],[str]) or None
     :returns: proxy version of the remote object
     :rtype: Pyro4.Proxy
 
@@ -543,7 +573,11 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090):
         # if there is no door on the other side, open one
         _copy_control(session, os.path.abspath(__file__), is_utility=True)
         run_remote_util(session, "remote_door", "share_local_object",
-                        object_name, host=host, port=port, detach=True)
+                        object_name, host=host, port=port,
+                        object_wl=object_wl,
+                        expose_wl=expose_wl,
+                        serialize_wl=serialize_wl,
+                        detach=True)
         output, attempts = "", 10
         for _ in range(attempts):
             output = session.get_output()
@@ -611,26 +645,35 @@ def get_remote_objects(session=None, host="localhost", port=0):
     return remote_objects
 
 
-def share_local_object(object_name, whitelist=None, host="localhost", port=9090):
+def share_local_object(object_name, host="localhost", port=9090,
+                       object_wl=None, expose_wl=None, serialize_wl=None):
     """
     Share a local object of the given name over the network.
 
     :param str object_name: name of the local object
-    :param whitelist: shared functions/classes of the local object as tuple
-                      pairs (module, function) or (module, class); whitelist
-                      all (do not filter) if None or empty
-    :type whitelist: [(str,str)] or None
     :param str host: ip address of the local name server
     :param port: port of the local sharing server
     :type port: int or str
+    :param object_wl: whitelist of allowed functions/classes of the local (typically
+                      module) object as a tuple pair (functions, classes);
+                      expose and autoproxy all (do not filter) if None or empty
+    :type object_wl: ([str],[str]) or None
+    :param expose_wl: extra classes to expose as a pair of full module class paths
+                      and general module paths to expose all classes from
+    :type expose_wl: ([str],[str]) or None
+    :param serialize_wl: exceptions to serialize as a pair of full module class paths
+                         and general module paths to serialize all exceptions from
+    :type serialize_wl: ([str],[str]) or None
 
     This function shares a custom object with whitelisted attributes through a
     custom implementation. It is more secure but more limited as functionality
     since it requires serialization extensions.
     """
     Pyro4.config.AUTOPROXY = True
-    Pyro4.config.REQUIRE_EXPOSE = False
     port = int(port) if isinstance(port, str) else port
+    object_wl = object_wl or ([], [])
+    expose_wl = expose_wl or ([], [])
+    serialize_wl = serialize_wl or ([], [])
 
     # pyro daemon
     try:
@@ -653,9 +696,7 @@ def share_local_object(object_name, whitelist=None, host="localhost", port=9090)
         LOG.debug("Pyro4 name server already started")
     # network unreachable and failed to locate the nameserver error
     except (OSError, Pyro4.errors.NamingError):
-        # noinspection PyPackageRequirements
-        from Pyro4 import naming
-        ns_uri, ns_daemon, _bc_server = naming.startNS(host=host, port=port)
+        ns_uri, ns_daemon, _bc_server = nameserver.start_ns(host=host, port=port)
         ns_server = Pyro4.Proxy(ns_uri)
         LOG.debug("Pyro4 name server started successfully with URI %s", ns_uri)
 
@@ -696,12 +737,31 @@ class ModuleObject:  # pylint: disable=R0903
         """Module wrapped for transferability."""
 
     for fname, fobj in inspect.getmembers(module, inspect.isfunction):
-        if not whitelist or (object_name, fname) in whitelist:
+        if not object_wl[0] or fname in object_wl[0]:
+            LOG.info("Autoproxying %s's function %s", object_name, fname)
             setattr(ModuleObject, fname, staticmethod(proxymethod(fobj)))
+            if not server.is_private_attribute(fname):
+                LOG.info("Exposing %s's function %s", object_name, fname)
+                try:
+                    server.expose(fobj)
+                except AttributeError as error:
+                    LOG.warning(error)
     for cname, cobj in inspect.getmembers(module, inspect.isclass):
-        if not whitelist or (object_name, cname) in whitelist:
+        if not object_wl[1] or cname in object_wl[1]:
+            LOG.info("Autoproxying %s's function %s", object_name, cname)
             setattr(ModuleObject, cname, staticmethod(proxymethod(cobj)))
+            if not server.is_private_attribute(cname):
+                LOG.info("Exposing %s's function %s", object_name, cname)
+                try:
+                    server.expose(cobj)
+                except AttributeError as error:
+                    LOG.warning(error)
+
     local_object = ModuleObject()
+    server.expose(ModuleObject)
+    # additional classes to expose and exceptions to serialize
+    expose_remote_classes(*expose_wl)
+    import_remote_exceptions(*serialize_wl)
 
     # we should register to the pyro daemon before entering its loop
     uri = pyro_daemon.register(local_object)
@@ -838,7 +898,7 @@ def import_remote_exceptions(exceptions=None, modules=None):
     :type exceptions: [str] or None
     :param modules: full module paths whose custom exceptions will first be
                     detected and then automatically imported (optional)
-    :type exceptions: [str] or None
+    :type modules: [str] or None
 
     The deserialization by our Pyro backend requires the full module paths to
     each exception or module in order to correctly detect the exception type.
@@ -885,3 +945,52 @@ def recreate_exception(class_name, class_dict):
     for exception in exceptions:
         # noinspection PyUnresolvedReferences
         Pyro4.util.SerializerBase.register_dict_to_class(exception, recreate_exception)
+
+
+def expose_remote_classes(classes=None, modules=None):
+    """
+    Make accessible all remote custom classes.
+
+    :param classes: full module path class names (optional)
+    :type classes: [str] or None
+    :param modules: full module paths whose custom classes will first be
+                    detected and then automatically exposed inclusive of
+                    parent classes and inheritance (optional)
+    :type modules: [str] or None
+    """
+    def list_module_classes(modstr):
+        imported_module = importlib.import_module(modstr)
+        module_classes = []
+        for name in imported_module.__dict__:
+            if not inspect.isclass(imported_module.__dict__[name]):
+                continue
+            module_classes.append(modstr + "." + name)
+        return module_classes
+
+    classes = [] if not classes else classes
+    modules = [] if not modules else modules
+    for module in modules:
+        classes += list_module_classes(module)
+    LOG.debug("Exposing the following classes (with proper inheritance): %s",
+              ", ".join(classes))
+
+    def get_class_from_name(clsstr):
+        module_name, class_name = clsstr.rsplit('.', 1)
+        module = importlib.import_module(module_name)
+        cls = getattr(module, class_name)
+        return cls
+
+    for cls in map(get_class_from_name, classes):
+        # the inheritance is inclusive of the class itself
+        for base_cls in cls.__mro__:
+            if base_cls in (object, int, float, bool, str, tuple, frozenset):
+                # known immutable classes should be skipped
+                continue
+            if inspect.ismethoddescriptor(base_cls):
+                # method descriptors should be skipped
+                continue
+            try:
+                server.expose(base_cls)
+            except (TypeError, AttributeError) as error:
+                logging.warning("Additional class exposing error: %s", error)
+                continue

tests/test_remote_door.py

@@ -19,8 +19,9 @@
 import re
 import shutil
 import unittest.mock
+import html
 
-from aexpect import remote_door
+from aexpect import client, remote, remote_door
 from aexpect.client import RemoteSession
 
 mock = unittest.mock
@@ -51,6 +52,16 @@ def setUp(self):
         if not os.path.isdir(remote_door.REMOTE_PYTHON_PATH):
             os.mkdir(remote_door.REMOTE_PYTHON_PATH)
 
+        self.has_remote_objects = hasattr(remote_door, "Pyro4")
+        if self.has_remote_objects:
+            self.Pyro4 = mock.patch(remote_door.Pyro4)
+            self.server = mock.patch(remote_door.server)
+            self.nameserver = mock.patch(remote_door.nameserver)
+        else:
+            self.Pyro4 = remote_door.Pyro4 = mock.MagicMock()
+            self.server = remote_door.server = mock.MagicMock()
+            self.nameserver = remote_door.nameserver = mock.MagicMock()
+
     def tearDown(self):
         for control_file in glob.glob("tmp*.control"):
             os.unlink(control_file)
@@ -63,6 +74,15 @@ def tearDown(self):
         os.rmdir(remote_door.REMOTE_PYTHON_PATH)
         self.session.close()
 
+        if self.has_remote_objects:
+            self.Pyro4.stop()
+            self.server.stop()
+            self.nameserver.stop()
+        else:
+            del remote_door.Pyro4
+            del remote_door.server
+            del remote_door.nameserver
+
     def test_run_remote_util(self):
         """Test that a remote utility runs properly."""
         result = remote_door.run_remote_util(self.session, "math", "gcd", 2, 3)
@@ -152,9 +172,8 @@ def test_get_remote_object(self):
         """Test that a remote object can be retrieved properly."""
         self.session = mock.MagicMock(name='session')
         self.session.client = "ssh"
-        remote_door.Pyro4 = mock.MagicMock()
-        disconnect = remote_door.Pyro4.errors.PyroError = Exception
-        remote_door.Pyro4.Proxy.side_effect = [disconnect("no such object"), mock.DEFAULT]
+        disconnect = self.Pyro4.errors.PyroError = Exception
+        self.Pyro4.Proxy.side_effect = [disconnect("no such object"), mock.DEFAULT]
         self.session.get_output.return_value = "Local object sharing ready\n"
         self.session.get_output.return_value += "RESULT = None\n"
 
@@ -180,17 +199,24 @@ def test_get_remote_object(self):
             control_lines = handle.readlines()
             self.assertIn("import remote_door\n", control_lines)
             self.assertIn("result = remote_door.share_local_object(r'html', "
-                          "host=r'testhost', port=4242)\n",
+                          "expose_wl=None, host=r'testhost', object_wl=None, "
+                          "port=4242, serialize_wl=None)\n",
                           control_lines)
 
-        # since the local run was face redo it here
-        remote_door.share_local_object("html", None, "testhost", 4242)
+        # since the local run was fake redo it here
+        self.server.is_private_attribute = lambda x: False
+        remote_door.share_local_object("html", "testhost", 4242)
+        self.server.expose.assert_called()
+        # TODO: to make the remote door usable with nearly no dependencies
+        # we use a lot of internal classes and functions to make sharing a
+        # local object possible but this makes testing these less accessible
+        self.server.expose.assert_any_call(html.escape)
+        self.server.expose.assert_any_call(html.unescape)
 
     def test_share_remote_objects(self):
         """Test that a remote object can be shared properly and remotely."""
         self.session = mock.MagicMock(name='session')
         self.session.client = "ssh"
-        remote_door.Pyro4 = mock.MagicMock()
 
         control_file = os.path.join(remote_door.REMOTE_CONTROL_DIR,
                                     "tmpxxxxxxxx.control")
@@ -212,12 +238,11 @@ def test_share_remote_objects(self):
 
     def test_import_remote_exceptions(self):
         """Test that selected remote exceptions are properly imported and deserialized."""
-        remote_door.Pyro4 = mock.MagicMock()
         preselected_exceptions = ["aexpect.remote.RemoteError",
                                   "aexpect.remote.LoginError",
                                   "aexpect.remote.TransferError"]
         remote_door.import_remote_exceptions(preselected_exceptions)
-        register_method = remote_door.Pyro4.util.SerializerBase.register_dict_to_class
+        register_method = self.Pyro4.util.SerializerBase.register_dict_to_class
         self.assertEqual(len(register_method.mock_calls), 3)
 
         def get_first_arg(call):
@@ -239,3 +264,34 @@ def get_first_arg(call):
         # assert some detected exceptions from the remote module
         self.assertIn("aexpect.remote.RemoteError", imported_classes)
         self.assertIn("aexpect.remote.UDPError", imported_classes)
+
+    def test_expose_remote_classes(self):
+        """Test that selected remote classes are properly exposed."""
+        preselected_classes = ["aexpect.client.ShellSession",
+                               "aexpect.remote.LoginError"]
+        remote_door.expose_remote_classes(preselected_classes)
+        expose_method = self.server.expose
+        self.assertEqual(expose_method.mock_calls[0],
+                         mock.call(client.ShellSession))
+        self.assertEqual(expose_method.mock_calls[1],
+                         mock.call(client.Expect))
+        self.assertEqual(expose_method.mock_calls[2],
+                         mock.call(client.Tail))
+        self.assertEqual(expose_method.mock_calls[3],
+                         mock.call(client.Spawn))
+        self.assertNotEqual(expose_method.mock_calls[4],
+                            mock.call(object))
+        self.assertEqual(expose_method.mock_calls[4],
+                         mock.call(remote.LoginError))
+        self.assertEqual(expose_method.mock_calls[5],
+                         mock.call(remote.RemoteError))
+
+        expose_method.reset_mock()
+        preselected_modules = ["aexpect.client"]
+        remote_door.expose_remote_classes([], modules=preselected_modules)
+        self.assertIn(mock.call(client.ExpectError),
+                      expose_method.mock_calls,
+                      "classes imported from elsewhere are exposed")
+        self.assertIn(mock.call(client.ShellSession),
+                      expose_method.mock_calls,
+                      "defined classes are exposed")