ipmock: Merge pull request #1244 from svinota/p8-mock-engine

rtnl mock engine

Bug-Url: #1244

docs/_static/custom.css

@@ -232,15 +232,20 @@ pre {
     color: #333333;
     line-height: 120%;
     border: 1px solid #ac9;
-    border-left: none;
-    border-right: none;
     border-radius: 10px;
 }
 
+div.highlight-python > div.highlight > pre {
+    padding-bottom: 2em;
+}
+
 div.highlight-none > div.highlight > pre {
-    border: none;
-    margin-top: -1.5em;
-    padding-top: 1.5em;
+    background-color: #fffeed;
+}
+
+div.highlight-python + div.highlight-none {
+    margin-top: -2em;
+    padding-left: 1em;
 }
 
 tt {

docs/usage.rst

@@ -1,26 +1,71 @@
 .. usage:
 
+.. testsetup:: *
+
+    from pyroute2 import config
+    config.mock_iproute = True
+
 Quickstart
 ==========
 
-Hello, world::
+"Hello world", sync API:
 
-    $ pip install pyroute2
+.. testcode::
 
-    $ cat example.py
     from pyroute2 import IPRoute
-    with IPRoute() as ipr:
-        print([x.get_attr('IFLA_IFNAME') for x in ipr.get_links()])
 
-    $ python example.py
-    ['lo', 'p6p1', 'wlan0', 'virbr0', 'virbr0-nic']
+    def main():
+        ipr = IPRoute()
+        for link in ipr.link("dump"):
+            print(link.get("ifname"), link.get("state"), link.get("address"))
+        ipr.close()
 
-Sockets
--------
+    main()
+
+
+.. testoutput::
+
+    lo up 00:00:00:00:00:00
+    eth0 up 52:54:00:72:58:b2
+
+"Hello world", async API:
+
+.. testcode::
+
+    import asyncio
+
+    from pyroute2 import AsyncIPRoute
+
+    async def main():
+        ipr = AsyncIPRoute()
+        async for link in await ipr.link("dump"):
+            print(link.get("ifname"), link.get("state"), link.get("address"))
+        await ipr.close()
+
+    asyncio.run(main())
+
+
+.. testoutput::
+
+    lo up 00:00:00:00:00:00
+    eth0 up 52:54:00:72:58:b2
+
+Netlink sockets
+---------------
 
-In the runtime pyroute2 socket objects behave as normal
-sockets. One can use them in the poll/select, one can
-call `recv()` and `sendmsg()`::
+Netlink sockets created with pyroute2 behave similarly to ordinary
+socket objects, but there are some key differences in how they
+handle data reception.
+
+At a low level, these sockets are monitored by the asyncio event
+loop, which means that direct use of `recv()` or `recvmsg()` is not
+supported. If you require such low-level functionality, you would
+need to modify the protocol class associated with the socket object.
+
+By default, the lowest-level API available for receiving data with
+pyroute2 sockets is the `get()` method.
+
+.. testcode::
 
     from pyroute2 import IPRoute
 
@@ -30,33 +75,28 @@ call `recv()` and `sendmsg()`::
     # subscribe to broadcast messages
     ipr.bind()
 
-    # wait for data (do not parse it)
-    data = ipr.recv(65535)
-
-    # parse received data
-    messages = ipr.marshal.parse(data)
+    # wait for parsed data
+    data = ipr.get()
 
-    # shortcut: recv() + parse()
-    #
-    # (under the hood is much more, but for
-    # simplicity it's enough to say so)
-    #
-    messages = ipr.get()
 
+... but pyroute2 objects have additional high level methods:
 
-But pyroute2 objects have a lot of methods, written to
-handle specific tasks::
+.. testcode::
 
     from pyroute2 import IPRoute
 
     # RTNL interface
     with IPRoute() as ipr:
+        # get IP addresses
+        for msg in ipr.addr("dump"):
+            addr = msg.get("address")
+            mask = msg.get("prefixlen")
+            print(f"{addr}/{mask}")
 
-        # get devices list
-        ipr.get_links()
+.. testoutput::
 
-        # get addresses
-        ipr.get_addr()
+    127.0.0.1/8
+    192.168.122.28/24
 
 Resource release
 ----------------
@@ -80,13 +120,15 @@ import signature.
     All other objects are also available for import, but they
     may change signatures in the next versions.
 
-E.g.::
+E.g.:
+
+.. testcode::
 
     # Import a pyroute2 class directly. In the next versions
     # the import signature can be changed, e.g., NetNS from
     # pyroute2.netns.nslink it can be moved somewhere else.
     #
-    from pyroute2.netns.nslink import NetNS
+    from pyroute2.iproute.linux import NetNS
     ns = NetNS('test')
 
     # Import the same class from root module. This signature
@@ -95,30 +137,3 @@ E.g.::
     #
     from pyroute2 import NetNS
     ns = NetNS('test')
-
-Special cases
-=============
-
-eventlet
---------
-
-The eventlet environment conflicts in some way with socket
-objects, and pyroute2 provides some workaround for that::
-
-    # import symbols
-    #
-    import eventlet
-    from pyroute2 import NetNS
-    from pyroute2.config.eventlet import eventlet_config
-
-    # setup the environment
-    eventlet.monkey_patch()
-    eventlet_config()
-
-    # run the code
-    ns = NetNS('nsname')
-    ns.get_routes()
-    ...
-
-This may help, but not always. In general, the pyroute2 library
-is not eventlet-friendly.

pyroute2/iproute/ipmock.py

@@ -5,10 +5,13 @@
 import struct
 from itertools import count
 
+from pyroute2.config import AF_NETLINK
 from pyroute2.lab import LAB_API
+from pyroute2.netlink import NLM_F_MULTI, NLMSG_DONE, nlmsg
 from pyroute2.netlink.core import Stats
 from pyroute2.netlink.exceptions import NetlinkError
 from pyroute2.netlink.nlsocket import NetlinkSocket
+from pyroute2.netlink.rtnl import RTM_GETADDR, RTM_GETLINK
 from pyroute2.netlink.rtnl.ifaddrmsg import ifaddrmsg
 from pyroute2.netlink.rtnl.ifinfmsg import ifinfmsg
 from pyroute2.netlink.rtnl.marshal import MarshalRtnl
@@ -490,7 +493,143 @@ def export(self):
 }
 
 
+class IPEngine:
+    '''Mock network objects database with the socket API.
+
+    WIP: work in progress.
+
+    A drop-in replacement to use instead of a low level RTNL socket.
+    Implements all the required socket properties and provides a
+    network objects database with RTNL protocol.
+
+    Example::
+
+        >>> ipe = IPEngine()
+        >>> ipr = IPRoute(use_socket=ipe)
+        >>> [ x.get('ifname') for x in ipr.link('dump') ]
+        ['lo', 'eth0']
+
+    '''
+
+    def __init__(self, sfamily=AF_NETLINK, stype=socket.SOCK_DGRAM, sproto=0):
+        self.marshal = MarshalRtnl()
+        self.database = copy.deepcopy(presets['default'])
+        self.loopback_r, self.loopback_w = socket.socketpair()
+        self._stype = stype
+        self._sfamily = sfamily
+        self._sproto = sproto
+        self.processors = {
+            RTM_GETADDR: self.RTM_GETADDR,
+            RTM_GETLINK: self.RTM_GETLINK,
+        }
+
+    def close(self):
+        self.loopback_r.close()
+        self.loopback_w.close()
+
+    def bind(self, address=None):
+        msg = ifinfmsg()
+        msg.load(self.database['links'][0])
+        msg.encode()
+        self.loopback_w.send(msg.data)
+
+    def fileno(self):
+        return self.loopback_r.fileno()
+
+    def recv(self, bufsize, flags=0):
+        return self.loopback_r.recv(bufsize, flags)
+
+    def recvfrom(self, bufsize, flags=0):
+        return self.loopback_r.recvfrom(bufsize, flags)
+
+    def recvmsg(self, bufsize, ancbufsize=0, flags=0):
+        return self.loopback_r.recvmsg(bufsize, ancbufsize, flags)
+
+    def recv_into(self, buffer, nbytes=0, flags=0):
+        return self.loopback_r.recv_into(buffer, nbytes, flags)
+
+    def recvfrom_into(self, buffer, nbytes=0, flags=0):
+        return self.loopback_r.recvfrom_into(buffer, nbytes, flags)
+
+    def recvmsg_into(self, buffers, ancbufsize=0, flags=0):
+        return self.loopback_r.recvmsg_into(buffers, ancbufsize, flags)
+
+    def send(self, data, flags=0):
+        return self.nl_handle(data)
+
+    def sendall(self, data, flags=0):
+        return self.nl_handle(data)
+
+    def sendto(self, data, flags, address=0):
+        return self.nl_handle(data)
+
+    def sendmsg(self, buffers, ancdata=None, flags=None, address=None):
+        raise NotImplementedError()
+
+    def setblocking(self, flag):
+        return self.loopback_r.setblocking(flag)
+
+    def getblocking(self):
+        return self.loopback_r.getblocking()
+
+    def getsockname(self):
+        return self.loopback_r.getsockname()
+
+    def getpeername(self):
+        return self.loopback_r.getpeername()
+
+    @property
+    def type(self):
+        return self._stype
+
+    @property
+    def family(self):
+        return self._sfamily
+
+    @property
+    def proto(self):
+        return self._sproto
+
+    def nl_handle(self, data):
+        for msg in self.marshal.parse(data):
+            key = msg['header']['type']
+            if key in self.processors:
+                self.processors[key](msg)
+        return len(data)
+
+    def nl_dump(self, registry, msg_class, tag):
+        for item in registry:
+            msg = msg_class()
+            msg.load(item.export())
+            msg['header']['flags'] = NLM_F_MULTI
+            msg['header']['sequence_number'] = tag
+            msg.encode()
+            yield msg
+
+    def RTM_GETADDR(self, msg_in):
+        tag = msg_in['header']['sequence_number']
+        for msg_out in self.nl_dump(self.database['addr'], ifaddrmsg, tag):
+            self.loopback_w.send(msg_out.data)
+        msg = nlmsg()
+        msg['header']['type'] = NLMSG_DONE
+        msg['header']['sequence_number'] = tag
+        msg.encode()
+        self.loopback_w.send(msg.data)
+
+    def RTM_GETLINK(self, msg_in):
+        tag = msg_in['header']['sequence_number']
+        for msg_out in self.nl_dump(self.database['links'], ifinfmsg, tag):
+            self.loopback_w.send(msg_out.data)
+        msg = nlmsg()
+        msg['header']['type'] = NLMSG_DONE
+        msg['header']['sequence_number'] = tag
+        msg.encode()
+        self.loopback_w.send(msg.data)
+
+
 class IPRoute(LAB_API, NetlinkSocket):
+    '''To be deprecated.'''
+
     def __init__(self, *argv, **kwarg):
         super().__init__()
         self.set_marshal(MarshalRtnl())

pyroute2/netlink/nlsocket.py

@@ -243,7 +243,7 @@ def __init__(
             'create_dummy': True,
             'provide_master': config.kernel[0] > 2,
         }
-        super().__init__(libc=libc)
+        super().__init__(libc=libc, use_socket=use_socket)
         self.marshal = Marshal()
         self.request_proxy = None
         self.batch = None

pyroute2/netlink/rtnl/iprsocket.py

@@ -2,6 +2,7 @@
 import sys
 
 from pyroute2 import config
+from pyroute2.iproute.ipmock import IPEngine
 from pyroute2.netlink import NETLINK_ROUTE, rtnl
 from pyroute2.netlink.nlsocket import AsyncNetlinkSocket, ChaoticNetlinkSocket
 from pyroute2.netlink.proxy import NetlinkProxy
@@ -89,6 +90,9 @@ def __init__(
         flags=os.O_CREAT,
         libc=None,
     ):
+        if config.mock_iproute:
+            use_socket = IPEngine()
+            netns = None
         self.marshal = MarshalRtnl()
         super().__init__(
             family=NETLINK_ROUTE,