diff --git a/aexpect/remote_door.py b/aexpect/remote_door.py index 70382fd..923ed3e 100644 --- a/aexpect/remote_door.py +++ b/aexpect/remote_door.py @@ -54,7 +54,7 @@ # disable import issues from optional dependencies or remote extra imports using # pylint: disable=E0401,C0415,W0212 -# disable too-many-* as we need them pylint: disable=R0912,R0913,R0914,R0915,C0302 +# disable too-many/few-* as we need them pylint: disable=R0903,R0912,R0913,R0914,R0915,R0917,C0302 # ..todo:: we could reduce the disabled issues after more significant refactoring import os @@ -70,8 +70,29 @@ # related to connectivity and perform further development on this utility. # os.environ["PYRO_LOGLEVEL"] = "DEBUG" try: - # noinspection PyPackageRequirements,PyUnresolvedReferences - import Pyro4 + try: + # noinspection PyPackageRequirements,PyUnresolvedReferences + from Pyro5.compatibility import Pyro4 + # noinspection PyPackageRequirements + from Pyro5 import server + # noinspection PyPackageRequirements + from Pyro5 import nameserver + except ImportError: + # noinspection PyPackageRequirements,PyUnresolvedReferences + import Pyro4 + # Pyro5 compatibility does not support submodules so we need separate handling + + class Server: + """Dummy class to replace a Pyro5 server.""" + def __init__(self): + self.__dict__ = {} + server = Server() + server.expose = Pyro4.expose # pylint: disable=W0201 + server.is_private_attribute = Pyro4.util.is_private_attribute # pylint: disable=W0201 + # noinspection PyPackageRequirements + from Pyro4 import naming as nameserver + nameserver.start_ns = nameserver.startNS + except ImportError: logging.warning("Remote object backend (Pyro4) not found, some functionality" " of the remote door will not be available") @@ -411,11 +432,13 @@ def set_subcontrol_parameter_dict(subcontrol, dict_name, value): @set_subcontrol -def set_subcontrol_parameter_object(subcontrol, value): +def set_subcontrol_parameter_object(subcontrol, object_type, value): """ Prepare a URI to remote params for the remote control file. :param str subcontrol: path to the original control + :param str object_type: type of the object described as module path text, + for example "collections.UserDict" :param value: control parameters value :type value: Params object :returns: path to the modified control @@ -438,7 +461,7 @@ def set_subcontrol_parameter_object(subcontrol, value): LOG.info("Sharing the test parameters over the network") Pyro4.config.AUTOPROXY = False - Pyro4.config.REQUIRE_EXPOSE = False + expose_remote_classes([object_type]) try: pyro_daemon = Pyro4.Daemon(host=host_ip, port=1437) LOG.debug("Pyro4 daemon started successfully") @@ -496,7 +519,8 @@ def run(self): self.pyro_daemon.requestLoop() -def get_remote_object(object_name, session=None, host="localhost", port=9090): +def get_remote_object(object_name, session=None, host="localhost", port=9090, + object_wl=None, expose_wl=None, serialize_wl=None): """ Get a data object (visual or other) executing remotely or share one if none is available, generating control files along @@ -508,6 +532,16 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): :type session: RemoteSession or None :param str host: ip address of the local sharing server :param int port: port of the local name server + :param object_wl: whitelist of allowed functions/classes of the local (typically + module) object as a tuple pair (functions, classes); + expose and autoproxy all (do not filter) if None or empty + :type object_wl: ([str],[str]) or None + :param expose_wl: extra classes to expose as a pair of full module class paths + and general module paths to expose all classes from + :type expose_wl: ([str],[str]) or None + :param serialize_wl: exceptions to serialize as a pair of full module class paths + and general module paths to serialize all exceptions from + :type serialize_wl: ([str],[str]) or None :returns: proxy version of the remote object :rtype: Pyro4.Proxy @@ -531,6 +565,10 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): This method does not rely on any static (template) controls in order to work because the remote door takes care to reach back the local one. """ + serialize_wl = serialize_wl or ([], []) + # additional exceptions to serialize + import_remote_exceptions(*serialize_wl) + # noinspection PyUnresolvedReferences try: remote_object = Pyro4.Proxy(f"PYRONAME:{object_name}@{host}:{port}") @@ -543,7 +581,10 @@ def get_remote_object(object_name, session=None, host="localhost", port=9090): # if there is no door on the other side, open one _copy_control(session, os.path.abspath(__file__), is_utility=True) run_remote_util(session, "remote_door", "share_local_object", - object_name, host=host, port=port, detach=True) + object_name, host=host, port=port, + object_wl=object_wl, + expose_wl=expose_wl, + detach=True) output, attempts = "", 10 for _ in range(attempts): output = session.get_output() @@ -611,26 +652,31 @@ def get_remote_objects(session=None, host="localhost", port=0): return remote_objects -def share_local_object(object_name, whitelist=None, host="localhost", port=9090): +def share_local_object(object_name, host="localhost", port=9090, + object_wl=None, expose_wl=None): """ Share a local object of the given name over the network. :param str object_name: name of the local object - :param whitelist: shared functions/classes of the local object as tuple - pairs (module, function) or (module, class); whitelist - all (do not filter) if None or empty - :type whitelist: [(str,str)] or None :param str host: ip address of the local name server :param port: port of the local sharing server :type port: int or str + :param object_wl: whitelist of allowed functions/classes of the local (typically + module) object as a tuple pair (functions, classes); + expose and autoproxy all (do not filter) if None or empty + :type object_wl: ([str],[str]) or None + :param expose_wl: extra classes to expose as a pair of full module class paths + and general module paths to expose all classes from + :type expose_wl: ([str],[str]) or None This function shares a custom object with whitelisted attributes through a custom implementation. It is more secure but more limited as functionality since it requires serialization extensions. """ Pyro4.config.AUTOPROXY = True - Pyro4.config.REQUIRE_EXPOSE = False port = int(port) if isinstance(port, str) else port + object_wl = object_wl or ([], []) + expose_wl = expose_wl or ([], []) # pyro daemon try: @@ -653,9 +699,7 @@ def share_local_object(object_name, whitelist=None, host="localhost", port=9090) LOG.debug("Pyro4 name server already started") # network unreachable and failed to locate the nameserver error except (OSError, Pyro4.errors.NamingError): - # noinspection PyPackageRequirements - from Pyro4 import naming - ns_uri, ns_daemon, _bc_server = naming.startNS(host=host, port=port) + ns_uri, ns_daemon, _bc_server = nameserver.start_ns(host=host, port=port) ns_server = Pyro4.Proxy(ns_uri) LOG.debug("Pyro4 name server started successfully with URI %s", ns_uri) @@ -696,12 +740,30 @@ class ModuleObject: # pylint: disable=R0903 """Module wrapped for transferability.""" for fname, fobj in inspect.getmembers(module, inspect.isfunction): - if not whitelist or (object_name, fname) in whitelist: + if not object_wl[0] or fname in object_wl[0]: + LOG.info("Autoproxying %s's function %s", object_name, fname) setattr(ModuleObject, fname, staticmethod(proxymethod(fobj))) + if not server.is_private_attribute(fname): + LOG.info("Exposing %s's function %s", object_name, fname) + try: + server.expose(fobj) + except AttributeError as error: + LOG.warning(error) for cname, cobj in inspect.getmembers(module, inspect.isclass): - if not whitelist or (object_name, cname) in whitelist: + if not object_wl[1] or cname in object_wl[1]: + LOG.info("Autoproxying %s's class %s", object_name, cname) setattr(ModuleObject, cname, staticmethod(proxymethod(cobj))) + if not server.is_private_attribute(cname): + LOG.info("Exposing %s's class %s", object_name, cname) + try: + server.expose(cobj) + except AttributeError as error: + LOG.warning(error) + local_object = ModuleObject() + server.expose(ModuleObject) + # additional classes to expose + expose_remote_classes(*expose_wl) # we should register to the pyro daemon before entering its loop uri = pyro_daemon.register(local_object) @@ -885,3 +947,52 @@ def recreate_exception(class_name, class_dict): for exception in exceptions: # noinspection PyUnresolvedReferences Pyro4.util.SerializerBase.register_dict_to_class(exception, recreate_exception) + + +def expose_remote_classes(classes=None, modules=None): + """ + Make accessible all remote custom classes. + + :param classes: full module path class names (optional) + :type classes: [str] or None + :param modules: full module paths whose custom classes will first be + detected and then automatically exposed inclusive of + parent classes and inheritance (optional) + :type modules: [str] or None + """ + def list_module_classes(modstr): + imported_module = importlib.import_module(modstr) + module_classes = [] + for name in imported_module.__dict__: + if not inspect.isclass(imported_module.__dict__[name]): + continue + module_classes.append(modstr + "." + name) + return module_classes + + classes = [] if not classes else classes + modules = [] if not modules else modules + for module in modules: + classes += list_module_classes(module) + LOG.debug("Exposing the following classes (with proper inheritance): %s", + ", ".join(classes)) + + def get_class_from_name(clsstr): + module_name, class_name = clsstr.rsplit('.', 1) + module = importlib.import_module(module_name) + cls = getattr(module, class_name) + return cls + + for cls in map(get_class_from_name, classes): + # the inheritance is inclusive of the class itself + for base_cls in cls.__mro__: + if base_cls in (object, int, float, bool, str, tuple, frozenset): + # known immutable classes should be skipped + continue + if inspect.ismethoddescriptor(base_cls): + # method descriptors should be skipped + continue + try: + server.expose(base_cls) + except (TypeError, AttributeError) as error: + logging.warning("Additional class exposing error: %s", error) + continue diff --git a/tests/test_remote_door.py b/tests/test_remote_door.py index 7d93520..61f656e 100644 --- a/tests/test_remote_door.py +++ b/tests/test_remote_door.py @@ -12,29 +12,30 @@ # Copyright: Intra2net AG and aexpect contributors # Author: Plamen Dimitrov # -# selftests pylint: disable=C0111,C0111,W0613,R0913 +# selftests pylint: disable=C0111,C0111,W0613,R0913,E1101 import os import glob import re import shutil import unittest.mock +import html -from aexpect import remote_door +from aexpect import client, remote, remote_door from aexpect.client import RemoteSession mock = unittest.mock # noinspection PyUnusedLocal -def _local_login(client, host, port, username, password, prompt, +def _local_login(_client, host, port, username, password, prompt, linesep="\n", log_filename=None, log_function=None, timeout=10, internal_timeout=10, interface=None): - return RemoteSession("sh", prompt=prompt, client=client) + return RemoteSession("sh", prompt=prompt, client=_client) # noinspection PyUnusedLocal -def _local_copy(address, client, username, password, port, local_path, +def _local_copy(address, _client, username, password, port, local_path, remote_path, limit="", log_filename=None, log_function=None, verbose=False, timeout=600, interface=None, filesize=None, directory=True): @@ -51,6 +52,16 @@ def setUp(self): if not os.path.isdir(remote_door.REMOTE_PYTHON_PATH): os.mkdir(remote_door.REMOTE_PYTHON_PATH) + self.has_remote_objects = hasattr(remote_door, "Pyro4") + if self.has_remote_objects: + self.pyro = mock.patch(remote_door.Pyro4) + self.server = mock.patch(remote_door.server) + self.nameserver = mock.patch(remote_door.nameserver) + else: + self.pyro = remote_door.Pyro4 = mock.MagicMock() + self.server = remote_door.server = mock.MagicMock() + self.nameserver = remote_door.nameserver = mock.MagicMock() + def tearDown(self): for control_file in glob.glob("tmp*.control"): os.unlink(control_file) @@ -63,6 +74,15 @@ def tearDown(self): os.rmdir(remote_door.REMOTE_PYTHON_PATH) self.session.close() + if self.has_remote_objects: + self.pyro.stop() + self.server.stop() + self.nameserver.stop() + else: + del remote_door.Pyro4 + del remote_door.server + del remote_door.nameserver + def test_run_remote_util(self): """Test that a remote utility runs properly.""" result = remote_door.run_remote_util(self.session, "math", "gcd", 2, 3) @@ -152,9 +172,8 @@ def test_get_remote_object(self): """Test that a remote object can be retrieved properly.""" self.session = mock.MagicMock(name='session') self.session.client = "ssh" - remote_door.Pyro4 = mock.MagicMock() - disconnect = remote_door.Pyro4.errors.PyroError = Exception - remote_door.Pyro4.Proxy.side_effect = [disconnect("no such object"), mock.DEFAULT] + disconnect = self.pyro.errors.PyroError = Exception + self.pyro.Proxy.side_effect = [disconnect("no such object"), mock.DEFAULT] self.session.get_output.return_value = "Local object sharing ready\n" self.session.get_output.return_value += "RESULT = None\n" @@ -180,17 +199,24 @@ def test_get_remote_object(self): control_lines = handle.readlines() self.assertIn("import remote_door\n", control_lines) self.assertIn("result = remote_door.share_local_object(r'html', " - "host=r'testhost', port=4242)\n", + "expose_wl=None, host=r'testhost', object_wl=None, " + "port=4242)\n", control_lines) - # since the local run was face redo it here - remote_door.share_local_object("html", None, "testhost", 4242) + # since the local run was fake redo it here + self.server.is_private_attribute = lambda x: False + remote_door.share_local_object("html", "testhost", 4242) + self.server.expose.assert_called() + # TODO: to make the remote door usable with nearly no dependencies + # we use a lot of internal classes and functions to make sharing a + # local object possible but this makes testing these less accessible + self.server.expose.assert_any_call(html.escape) + self.server.expose.assert_any_call(html.unescape) def test_share_remote_objects(self): """Test that a remote object can be shared properly and remotely.""" self.session = mock.MagicMock(name='session') self.session.client = "ssh" - remote_door.Pyro4 = mock.MagicMock() control_file = os.path.join(remote_door.REMOTE_CONTROL_DIR, "tmpxxxxxxxx.control") @@ -212,12 +238,11 @@ def test_share_remote_objects(self): def test_import_remote_exceptions(self): """Test that selected remote exceptions are properly imported and deserialized.""" - remote_door.Pyro4 = mock.MagicMock() preselected_exceptions = ["aexpect.remote.RemoteError", "aexpect.remote.LoginError", "aexpect.remote.TransferError"] remote_door.import_remote_exceptions(preselected_exceptions) - register_method = remote_door.Pyro4.util.SerializerBase.register_dict_to_class + register_method = self.pyro.util.SerializerBase.register_dict_to_class self.assertEqual(len(register_method.mock_calls), 3) def get_first_arg(call): @@ -239,3 +264,34 @@ def get_first_arg(call): # assert some detected exceptions from the remote module self.assertIn("aexpect.remote.RemoteError", imported_classes) self.assertIn("aexpect.remote.UDPError", imported_classes) + + def test_expose_remote_classes(self): + """Test that selected remote classes are properly exposed.""" + preselected_classes = ["aexpect.client.ShellSession", + "aexpect.remote.LoginError"] + remote_door.expose_remote_classes(preselected_classes) + expose_method = self.server.expose + self.assertEqual(expose_method.mock_calls[0], + mock.call(client.ShellSession)) + self.assertEqual(expose_method.mock_calls[1], + mock.call(client.Expect)) + self.assertEqual(expose_method.mock_calls[2], + mock.call(client.Tail)) + self.assertEqual(expose_method.mock_calls[3], + mock.call(client.Spawn)) + self.assertNotEqual(expose_method.mock_calls[4], + mock.call(object)) + self.assertEqual(expose_method.mock_calls[4], + mock.call(remote.LoginError)) + self.assertEqual(expose_method.mock_calls[5], + mock.call(remote.RemoteError)) + + expose_method.reset_mock() + preselected_modules = ["aexpect.client"] + remote_door.expose_remote_classes([], modules=preselected_modules) + self.assertIn(mock.call(client.ExpectError), + expose_method.mock_calls, + "classes imported from elsewhere are exposed") + self.assertIn(mock.call(client.ShellSession), + expose_method.mock_calls, + "defined classes are exposed")