From 84ecd049ebf6c8665f21f18ef3cb9b68f5ee3b79 Mon Sep 17 00:00:00 2001 From: Plamen Dimitrov Date: Tue, 15 Oct 2024 15:38:41 +0300 Subject: [PATCH 1/2] Fix wrongly reused argument type for imported remote exceptions The documented type was a result of a copy paste. Signed-off-by: Plamen Dimitrov --- aexpect/remote_door.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aexpect/remote_door.py b/aexpect/remote_door.py index 36b4a13..70382fd 100644 --- a/aexpect/remote_door.py +++ b/aexpect/remote_door.py @@ -838,7 +838,7 @@ def import_remote_exceptions(exceptions=None, modules=None): :type exceptions: [str] or None :param modules: full module paths whose custom exceptions will first be detected and then automatically imported (optional) - :type exceptions: [str] or None + :type modules: [str] or None The deserialization by our Pyro backend requires the full module paths to each exception or module in order to correctly detect the exception type. From 8f4c5ccd707505481e9ee5a93382a860774e878a Mon Sep 17 00:00:00 2001 From: Plamen Dimitrov Date: Tue, 16 Jul 2024 03:33:03 +0700 Subject: [PATCH 2/2] Provide Pyro5 compatibility to the remote door utility The main adaptation that was needed comes down to dropping the previous expose-free approach and dynamically exposing each relevant function and class in addition to the previous autoproxy steps. To allow for modules and classes that the shared remote object has not explicitly imported or are not detectable as needing exposing, the previous whitelist argument has been extended to allow exposing predefined classes and serializing predefined exceptions that could be used and raised during remote use. Signed-off-by: Plamen Dimitrov --- aexpect/remote_door.py | 147 +++++++++++++++++++++++++++++++++----- tests/test_remote_door.py | 84 ++++++++++++++++++---- 2 files changed, 199 insertions(+), 32 deletions(-) diff --git a/aexpect/remote_door.py b/aexpect/remote_door.py index 70382fd..923ed3e 100644 --- a/aexpect/remote_door.py +++ b/aexpect/remote_door.py @@ -54,7 +54,7 @@ # disable import issues from optional dependencies or remote extra imports using # pylint: disable=E0401,C0415,W0212 -# disable too-many-* as we need them pylint: disable=R0912,R0913,R0914,R0915,C0302 +# disable too-many/few-* as we need them pylint: disable=R0903,R0912,R0913,R0914,R0915,R0917,C0302 # ..todo:: we could reduce the disabled issues after more significant refactoring import os @@ -70,8 +70,29 @@ # related to connectivity and perform further development on this utility. # os.environ["PYRO_LOGLEVEL"] = "DEBUG" try: - # noinspection PyPackageRequirements,PyUnresolvedReferences - import Pyro4 + try: + # noinspection PyPackageRequirements,PyUnresolvedReferences + from Pyro5.compatibility import Pyro4 + # noinspection PyPackageRequirements + from Pyro5 import server + # noinspection PyPackageRequirements + from Pyro5 import nameserver + except ImportError: + # noinspection PyPackageRequirements,PyUnresolvedReferences + import Pyro4 + # Pyro5 compatibility does not support submodules so we need separate handling + + class Server: + """Dummy class to replace a Pyro5 server.""" + def __init__(self): + self.__dict__ = {} + server = Server() + server.expose = Pyro4.expose # pylint: disable=W0201 + server.is_private_attribute = Pyro4.util.is_private_attribute # pylint: disable=W0201 + # noinspection PyPackageRequirements + from Pyro4 import naming as nameserver + nameserver.start_ns = nameserver.startNS + except ImportError: logging.warning("Remote object backend (Pyro4) not found, some functionality" " of the remote door will not be available") @@ -411,11 +432,13 @@ def set_subcontrol_parameter_dict(subcontrol, dict_name, value): @set_subcontrol -def set_subcontrol_parameter_object(subcontrol, value): +def set_subcontrol_parameter_object(subcontrol, object_type, value): """ Prepare a URI to remote params for the remote control file. :param str subcontrol: path to the original control + :param str object_type: type of the object described as module path text, + for example "collections.UserDict" :param value: control parameters value :type value: Params object :returns: path to the modified control @@ -438,7 +461,7 @@ def set_subcontrol_parameter_object(subcontrol, value): LOG.info("Sharing the test parameters over the network") Pyro4.config.AUTOPROXY = False - Pyro4.config.REQUIRE_EXPOSE = False + expose_remote_classes([object_type]) try: pyro_daemon = Pyro4.Daemon(host=host_ip, port=1437) LOG.debug("Pyro4 daemon started successfully") @@ -496,7 +519,8 @@ def run(self): self.pyro_daemon.requestLoop() -def get_remote_object(object_name, session=None, host="localhost", port=9090): +def get_remote_object(object_name, session=None, host="localhost", port=9090, + object_wl=None, expose_wl=None, serialize_wl=None): """ Get a data object (visual or other) executing remotely or share one if none is available, generating control files along @@ -508,6 +532,16 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): :type session: RemoteSession or None :param str host: ip address of the local sharing server :param int port: port of the local name server + :param object_wl: whitelist of allowed functions/classes of the local (typically + module) object as a tuple pair (functions, classes); + expose and autoproxy all (do not filter) if None or empty + :type object_wl: ([str],[str]) or None + :param expose_wl: extra classes to expose as a pair of full module class paths + and general module paths to expose all classes from + :type expose_wl: ([str],[str]) or None + :param serialize_wl: exceptions to serialize as a pair of full module class paths + and general module paths to serialize all exceptions from + :type serialize_wl: ([str],[str]) or None :returns: proxy version of the remote object :rtype: Pyro4.Proxy @@ -531,6 +565,10 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): This method does not rely on any static (template) controls in order to work because the remote door takes care to reach back the local one. """ + serialize_wl = serialize_wl or ([], []) + # additional exceptions to serialize + import_remote_exceptions(*serialize_wl) + # noinspection PyUnresolvedReferences try: remote_object = Pyro4.Proxy(f"PYRONAME:{object_name}@{host}:{port}") @@ -543,7 +581,10 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): # if there is no door on the other side, open one _copy_control(session, os.path.abspath(__file__), is_utility=True) run_remote_util(session, "remote_door", "share_local_object", - object_name, host=host, port=port, detach=True) + object_name, host=host, port=port, + object_wl=object_wl, + expose_wl=expose_wl, + detach=True) output, attempts = "", 10 for _ in range(attempts): output = session.get_output() @@ -611,26 +652,31 @@ def get_remote_objects(session=None, host="localhost", port=0): return remote_objects -def share_local_object(object_name, whitelist=None, host="localhost", port=9090): +def share_local_object(object_name, host="localhost", port=9090, + object_wl=None, expose_wl=None): """ Share a local object of the given name over the network. :param str object_name: name of the local object - :param whitelist: shared functions/classes of the local object as tuple - pairs (module, function) or (module, class); whitelist - all (do not filter) if None or empty - :type whitelist: [(str,str)] or None :param str host: ip address of the local name server :param port: port of the local sharing server :type port: int or str + :param object_wl: whitelist of allowed functions/classes of the local (typically + module) object as a tuple pair (functions, classes); + expose and autoproxy all (do not filter) if None or empty + :type object_wl: ([str],[str]) or None + :param expose_wl: extra classes to expose as a pair of full module class paths + and general module paths to expose all classes from + :type expose_wl: ([str],[str]) or None This function shares a custom object with whitelisted attributes through a custom implementation. It is more secure but more limited as functionality since it requires serialization extensions. """ Pyro4.config.AUTOPROXY = True - Pyro4.config.REQUIRE_EXPOSE = False port = int(port) if isinstance(port, str) else port + object_wl = object_wl or ([], []) + expose_wl = expose_wl or ([], []) # pyro daemon try: @@ -653,9 +699,7 @@ def share_local_object(object_name, whitelist=None, host="localhost", port=9090) LOG.debug("Pyro4 name server already started") # network unreachable and failed to locate the nameserver error except (OSError, Pyro4.errors.NamingError): - # noinspection PyPackageRequirements - from Pyro4 import naming - ns_uri, ns_daemon, _bc_server = naming.startNS(host=host, port=port) + ns_uri, ns_daemon, _bc_server = nameserver.start_ns(host=host, port=port) ns_server = Pyro4.Proxy(ns_uri) LOG.debug("Pyro4 name server started successfully with URI %s", ns_uri) @@ -696,12 +740,30 @@ class ModuleObject: # pylint: disable=R0903 """Module wrapped for transferability.""" for fname, fobj in inspect.getmembers(module, inspect.isfunction): - if not whitelist or (object_name, fname) in whitelist: + if not object_wl[0] or fname in object_wl[0]: + LOG.info("Autoproxying %s's function %s", object_name, fname) setattr(ModuleObject, fname, staticmethod(proxymethod(fobj))) + if not server.is_private_attribute(fname): + LOG.info("Exposing %s's function %s", object_name, fname) + try: + server.expose(fobj) + except AttributeError as error: + LOG.warning(error) for cname, cobj in inspect.getmembers(module, inspect.isclass): - if not whitelist or (object_name, cname) in whitelist: + if not object_wl[1] or cname in object_wl[1]: + LOG.info("Autoproxying %s's class %s", object_name, cname) setattr(ModuleObject, cname, staticmethod(proxymethod(cobj))) + if not server.is_private_attribute(cname): + LOG.info("Exposing %s's class %s", object_name, cname) + try: + server.expose(cobj) + except AttributeError as error: + LOG.warning(error) + local_object = ModuleObject() + server.expose(ModuleObject) + # additional classes to expose + expose_remote_classes(*expose_wl) # we should register to the pyro daemon before entering its loop uri = pyro_daemon.register(local_object) @@ -885,3 +947,52 @@ def recreate_exception(class_name, class_dict): for exception in exceptions: # noinspection PyUnresolvedReferences Pyro4.util.SerializerBase.register_dict_to_class(exception, recreate_exception) + + +def expose_remote_classes(classes=None, modules=None): + """ + Make accessible all remote custom classes. + + :param classes: full module path class names (optional) + :type classes: [str] or None + :param modules: full module paths whose custom classes will first be + detected and then automatically exposed inclusive of + parent classes and inheritance (optional) + :type modules: [str] or None + """ + def list_module_classes(modstr): + imported_module = importlib.import_module(modstr) + module_classes = [] + for name in imported_module.__dict__: + if not inspect.isclass(imported_module.__dict__[name]): + continue + module_classes.append(modstr + "." + name) + return module_classes + + classes = [] if not classes else classes + modules = [] if not modules else modules + for module in modules: + classes += list_module_classes(module) + LOG.debug("Exposing the following classes (with proper inheritance): %s", + ", ".join(classes)) + + def get_class_from_name(clsstr): + module_name, class_name = clsstr.rsplit('.', 1) + module = importlib.import_module(module_name) + cls = getattr(module, class_name) + return cls + + for cls in map(get_class_from_name, classes): + # the inheritance is inclusive of the class itself + for base_cls in cls.__mro__: + if base_cls in (object, int, float, bool, str, tuple, frozenset): + # known immutable classes should be skipped + continue + if inspect.ismethoddescriptor(base_cls): + # method descriptors should be skipped + continue + try: + server.expose(base_cls) + except (TypeError, AttributeError) as error: + logging.warning("Additional class exposing error: %s", error) + continue diff --git a/tests/test_remote_door.py b/tests/test_remote_door.py index 7d93520..61f656e 100644 --- a/tests/test_remote_door.py +++ b/tests/test_remote_door.py @@ -12,29 +12,30 @@ # Copyright: Intra2net AG and aexpect contributors # Author: Plamen Dimitrov # -# selftests pylint: disable=C0111,C0111,W0613,R0913 +# selftests pylint: disable=C0111,C0111,W0613,R0913,E1101 import os import glob import re import shutil import unittest.mock +import html -from aexpect import remote_door +from aexpect import client, remote, remote_door from aexpect.client import RemoteSession mock = unittest.mock # noinspection PyUnusedLocal -def _local_login(client, host, port, username, password, prompt, +def _local_login(_client, host, port, username, password, prompt, linesep="\n", log_filename=None, log_function=None, timeout=10, internal_timeout=10, interface=None): - return RemoteSession("sh", prompt=prompt, client=client) + return RemoteSession("sh", prompt=prompt, client=_client) # noinspection PyUnusedLocal -def _local_copy(address, client, username, password, port, local_path, +def _local_copy(address, _client, username, password, port, local_path, remote_path, limit="", log_filename=None, log_function=None, verbose=False, timeout=600, interface=None, filesize=None, directory=True): @@ -51,6 +52,16 @@ def setUp(self): if not os.path.isdir(remote_door.REMOTE_PYTHON_PATH): os.mkdir(remote_door.REMOTE_PYTHON_PATH) + self.has_remote_objects = hasattr(remote_door, "Pyro4") + if self.has_remote_objects: + self.pyro = mock.patch(remote_door.Pyro4) + self.server = mock.patch(remote_door.server) + self.nameserver = mock.patch(remote_door.nameserver) + else: + self.pyro = remote_door.Pyro4 = mock.MagicMock() + self.server = remote_door.server = mock.MagicMock() + self.nameserver = remote_door.nameserver = mock.MagicMock() + def tearDown(self): for control_file in glob.glob("tmp*.control"): os.unlink(control_file) @@ -63,6 +74,15 @@ def tearDown(self): os.rmdir(remote_door.REMOTE_PYTHON_PATH) self.session.close() + if self.has_remote_objects: + self.pyro.stop() + self.server.stop() + self.nameserver.stop() + else: + del remote_door.Pyro4 + del remote_door.server + del remote_door.nameserver + def test_run_remote_util(self): """Test that a remote utility runs properly.""" result = remote_door.run_remote_util(self.session, "math", "gcd", 2, 3) @@ -152,9 +172,8 @@ def test_get_remote_object(self): """Test that a remote object can be retrieved properly.""" self.session = mock.MagicMock(name='session') self.session.client = "ssh" - remote_door.Pyro4 = mock.MagicMock() - disconnect = remote_door.Pyro4.errors.PyroError = Exception - remote_door.Pyro4.Proxy.side_effect = [disconnect("no such object"), mock.DEFAULT] + disconnect = self.pyro.errors.PyroError = Exception + self.pyro.Proxy.side_effect = [disconnect("no such object"), mock.DEFAULT] self.session.get_output.return_value = "Local object sharing ready\n" self.session.get_output.return_value += "RESULT = None\n" @@ -180,17 +199,24 @@ def test_get_remote_object(self): control_lines = handle.readlines() self.assertIn("import remote_door\n", control_lines) self.assertIn("result = remote_door.share_local_object(r'html', " - "host=r'testhost', port=4242)\n", + "expose_wl=None, host=r'testhost', object_wl=None, " + "port=4242)\n", control_lines) - # since the local run was face redo it here - remote_door.share_local_object("html", None, "testhost", 4242) + # since the local run was fake redo it here + self.server.is_private_attribute = lambda x: False + remote_door.share_local_object("html", "testhost", 4242) + self.server.expose.assert_called() + # TODO: to make the remote door usable with nearly no dependencies + # we use a lot of internal classes and functions to make sharing a + # local object possible but this makes testing these less accessible + self.server.expose.assert_any_call(html.escape) + self.server.expose.assert_any_call(html.unescape) def test_share_remote_objects(self): """Test that a remote object can be shared properly and remotely.""" self.session = mock.MagicMock(name='session') self.session.client = "ssh" - remote_door.Pyro4 = mock.MagicMock() control_file = os.path.join(remote_door.REMOTE_CONTROL_DIR, "tmpxxxxxxxx.control") @@ -212,12 +238,11 @@ def test_share_remote_objects(self): def test_import_remote_exceptions(self): """Test that selected remote exceptions are properly imported and deserialized.""" - remote_door.Pyro4 = mock.MagicMock() preselected_exceptions = ["aexpect.remote.RemoteError", "aexpect.remote.LoginError", "aexpect.remote.TransferError"] remote_door.import_remote_exceptions(preselected_exceptions) - register_method = remote_door.Pyro4.util.SerializerBase.register_dict_to_class + register_method = self.pyro.util.SerializerBase.register_dict_to_class self.assertEqual(len(register_method.mock_calls), 3) def get_first_arg(call): @@ -239,3 +264,34 @@ def get_first_arg(call): # assert some detected exceptions from the remote module self.assertIn("aexpect.remote.RemoteError", imported_classes) self.assertIn("aexpect.remote.UDPError", imported_classes) + + def test_expose_remote_classes(self): + """Test that selected remote classes are properly exposed.""" + preselected_classes = ["aexpect.client.ShellSession", + "aexpect.remote.LoginError"] + remote_door.expose_remote_classes(preselected_classes) + expose_method = self.server.expose + self.assertEqual(expose_method.mock_calls[0], + mock.call(client.ShellSession)) + self.assertEqual(expose_method.mock_calls[1], + mock.call(client.Expect)) + self.assertEqual(expose_method.mock_calls[2], + mock.call(client.Tail)) + self.assertEqual(expose_method.mock_calls[3], + mock.call(client.Spawn)) + self.assertNotEqual(expose_method.mock_calls[4], + mock.call(object)) + self.assertEqual(expose_method.mock_calls[4], + mock.call(remote.LoginError)) + self.assertEqual(expose_method.mock_calls[5], + mock.call(remote.RemoteError)) + + expose_method.reset_mock() + preselected_modules = ["aexpect.client"] + remote_door.expose_remote_classes([], modules=preselected_modules) + self.assertIn(mock.call(client.ExpectError), + expose_method.mock_calls, + "classes imported from elsewhere are exposed") + self.assertIn(mock.call(client.ShellSession), + expose_method.mock_calls, + "defined classes are exposed")