test: add tests for migration FSM paths

Signed-off-by: William Henderson <william.henderson@nutanix.com>

test/py/libvfio_user.py

@@ -197,11 +197,22 @@ def is_32bit():
 
 VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP = (1 << 0)
 
+# enum vfio_user_device_mig_state
+VFIO_USER_DEVICE_STATE_ERROR = 0
+VFIO_USER_DEVICE_STATE_STOP = 1
+VFIO_USER_DEVICE_STATE_RUNNING = 2
+VFIO_USER_DEVICE_STATE_STOP_COPY = 3
+VFIO_USER_DEVICE_STATE_RESUMING = 4
+VFIO_USER_DEVICE_STATE_RUNNING_P2P = 5
+VFIO_USER_DEVICE_STATE_PRE_COPY = 6
+VFIO_USER_DEVICE_STATE_PRE_COPY_P2P = 7
+
 VFIO_DEVICE_FEATURE_MASK = 0xffff
 VFIO_DEVICE_FEATURE_GET = (1 << 16)
 VFIO_DEVICE_FEATURE_SET = (1 << 17)
 VFIO_DEVICE_FEATURE_PROBE = (1 << 18)
 
+VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE = 2
 VFIO_DEVICE_FEATURE_DMA_LOGGING_START = 6
 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP = 7
 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT = 8
@@ -210,6 +221,13 @@ def is_32bit():
 VFIO_USER_IO_FD_TYPE_IOREGIONFD = 1
 VFIO_USER_IO_FD_TYPE_IOEVENTFD_SHADOW = 2
 
+# enum vfu_migr_state_t
+VFU_MIGR_STATE_STOP = 0
+VFU_MIGR_STATE_RUNNING = 1
+VFU_MIGR_STATE_STOP_AND_COPY = 2
+VFU_MIGR_STATE_PRE_COPY = 3
+VFU_MIGR_STATE_RESUME = 4
+
 
 # enum vfu_dev_irq_type
 VFU_DEV_INTX_IRQ = 0
@@ -556,6 +574,14 @@ class vfio_user_device_feature(Structure):
     ]
 
 
+class vfio_user_device_feature_mig_state(Structure):
+    _pack_ = 1
+    _fields_ = [
+        ("device_state", c.c_uint32),
+        ("data_fd", c.c_uint32),
+    ]
+
+
 class vfio_user_device_feature_dma_logging_control(Structure):
     _pack_ = 1
     _fields_ = [

test/py/meson.build

@@ -37,6 +37,7 @@ python_tests = [
     'test_dma_map.py',
     'test_dma_unmap.py',
     'test_irq_trigger.py',
+    'test_migration.py',
     'test_negotiate.py',
     'test_pci_caps.py',
     'test_pci_ext_caps.py',

test/py/test_migration.py

@@ -0,0 +1,248 @@
+#
+# Copyright (c) 2021 Nutanix Inc. All rights reserved.
+#
+# Authors: John Levon <john.levon@nutanix.com>
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#      * Redistributions of source code must retain the above copyright
+#        notice, this list of conditions and the following disclaimer.
+#      * Redistributions in binary form must reproduce the above copyright
+#        notice, this list of conditions and the following disclaimer in the
+#        documentation and/or other materials provided with the distribution.
+#      * Neither the name of Nutanix nor the names of its contributors may be
+#        used to endorse or promote products derived from this software without
+#        specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+#  ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+#  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+#  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+#  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+#  OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+#  DAMAGE.
+#
+
+from libvfio_user import *
+from collections import deque
+
+ctx = None
+sock = None
+current_state = None
+path = []
+
+
+UNREACHABLE_STATES = {
+    VFIO_USER_DEVICE_STATE_ERROR,
+    VFIO_USER_DEVICE_STATE_PRE_COPY_P2P,
+    VFIO_USER_DEVICE_STATE_RUNNING_P2P
+}
+
+
+@transition_cb_t
+def migr_trans_cb(_ctx, state):
+    global current_state, path
+
+    if state == VFU_MIGR_STATE_STOP:
+        state = VFIO_USER_DEVICE_STATE_STOP
+    elif state == VFU_MIGR_STATE_RUNNING:
+        state = VFIO_USER_DEVICE_STATE_RUNNING
+    elif state == VFU_MIGR_STATE_STOP_AND_COPY:
+        state = VFIO_USER_DEVICE_STATE_STOP_COPY
+    elif state == VFU_MIGR_STATE_RESUME:
+        state = VFIO_USER_DEVICE_STATE_RESUMING
+    elif state == VFU_MIGR_STATE_PRE_COPY:
+        state = VFIO_USER_DEVICE_STATE_PRE_COPY
+    else:
+        assert False
+
+    current_state = state
+
+    path.append(state)
+
+    return 0
+
+
+@read_data_cb_t
+def migr_read_data_cb(_ctx, _buf, _count, _offset):
+    return
+
+
+@write_data_cb_t
+def migr_write_data_cb(_ctx, _buf, _count, _offset):
+    return
+
+
+def test_migration_setup():
+    global ctx, sock
+
+    ctx = vfu_create_ctx(flags=LIBVFIO_USER_FLAG_ATTACH_NB)
+    assert ctx is not None
+
+    cbs = vfu_migration_callbacks_t()
+    cbs.version = VFU_MIGR_CALLBACKS_VERS
+    cbs.transition = migr_trans_cb
+    cbs.read_data = migr_read_data_cb
+    cbs.write_data = migr_write_data_cb
+
+    ret = vfu_setup_device_migration_callbacks(ctx, 0, cbs)
+    assert ret == 0
+
+    vfu_setup_device_quiesce_cb(ctx)
+
+    ret = vfu_realize_ctx(ctx)
+    assert ret == 0
+
+    sock = connect_client(ctx)
+
+
+def get_server_shortest_path(a, b, expectA=0, expectB=0):
+    global ctx, sock, path
+
+    feature = vfio_user_device_feature(
+        argsz=len(vfio_user_device_feature()) +
+            len(vfio_user_device_feature_mig_state()),
+        flags=VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE
+    )
+
+    if current_state == VFIO_USER_DEVICE_STATE_STOP_COPY and \
+            a == VFIO_USER_DEVICE_STATE_PRE_COPY:
+        # The transition STOP_COPY -> PRE_COPY is explicitly blocked so we
+        # advance one state to get around this in order to set up the test.
+        payload = vfio_user_device_feature_mig_state(
+            device_state=VFIO_USER_DEVICE_STATE_STOP
+        )
+        msg(ctx, sock, VFIO_USER_DEVICE_FEATURE,
+            bytes(feature) + bytes(payload))
+
+    payload = vfio_user_device_feature_mig_state(device_state=a)
+    msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload),
+        expect=expectA)
+
+    if expectA != 0:
+        return None
+
+    path = []
+
+    payload = vfio_user_device_feature_mig_state(device_state=b)
+    msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload),
+        expect=expectB)
+
+    return path.copy()
+
+
+def test_migration_shortest_state_transition_paths():
+    """
+    The spec dictates that complex state transitions are to be implemented as
+    combinations of the defined direct transitions, with the path selected
+    according to the following rules:
+
+    - Select the shortest path.
+    - The path cannot have saving group states as interior arcs, only start/end
+      states.
+
+    This test implements a breadth-first search to ensure that the paths taken
+    by the implementation correctly follow these rules.
+    """
+
+    global ctx, sock
+
+    # states (vertices)
+    V = {VFIO_USER_DEVICE_STATE_ERROR, VFIO_USER_DEVICE_STATE_STOP,
+         VFIO_USER_DEVICE_STATE_RUNNING, VFIO_USER_DEVICE_STATE_STOP_COPY,
+         VFIO_USER_DEVICE_STATE_RESUMING, VFIO_USER_DEVICE_STATE_RUNNING_P2P,
+         VFIO_USER_DEVICE_STATE_PRE_COPY, VFIO_USER_DEVICE_STATE_PRE_COPY_P2P}
+
+    # allowed direct transitions (edges)
+    E = {
+        VFIO_USER_DEVICE_STATE_ERROR: set(),
+        VFIO_USER_DEVICE_STATE_STOP: {
+            VFIO_USER_DEVICE_STATE_RUNNING,
+            VFIO_USER_DEVICE_STATE_STOP_COPY,
+            VFIO_USER_DEVICE_STATE_RESUMING
+        },
+        VFIO_USER_DEVICE_STATE_RUNNING: {
+            VFIO_USER_DEVICE_STATE_STOP,
+            VFIO_USER_DEVICE_STATE_PRE_COPY
+        },
+        VFIO_USER_DEVICE_STATE_STOP_COPY: {VFIO_USER_DEVICE_STATE_STOP},
+        VFIO_USER_DEVICE_STATE_RESUMING: {VFIO_USER_DEVICE_STATE_STOP},
+        VFIO_USER_DEVICE_STATE_RUNNING_P2P: set(),
+        VFIO_USER_DEVICE_STATE_PRE_COPY: {
+            VFIO_USER_DEVICE_STATE_RUNNING,
+            VFIO_USER_DEVICE_STATE_STOP_COPY
+        },
+        VFIO_USER_DEVICE_STATE_PRE_COPY_P2P: set()
+    }
+
+    # "saving states" which cannot be internal arcs
+    S = {VFIO_USER_DEVICE_STATE_PRE_COPY, VFIO_USER_DEVICE_STATE_STOP_COPY}
+
+    for source in V:
+        back = {v: None for v in V}
+        queue = deque([(source, None)])
+
+        while len(queue) > 0:
+            (curr, prev) = queue.popleft()
+            back[curr] = prev
+            for nxt in E[curr]:
+                if back[nxt] is None and (curr == source or curr not in S):
+                    queue.append((nxt, curr))
+
+        for target in V:
+            if source == VFIO_USER_DEVICE_STATE_STOP_COPY \
+                    and target == VFIO_USER_DEVICE_STATE_PRE_COPY:
+                # test for this transition being blocked in a separate test
+                continue
+
+            if back[target] is not None:
+                seq = deque([])
+                curr = target
+                while curr != source:
+                    seq.appendleft(curr)
+                    curr = back[curr]
+
+                print(f"{source}->{target}: test wants   {seq}")
+
+                server_seq = get_server_shortest_path(source, target)
+
+                print(f"{source}->{target}: server wants {server_seq}")
+
+                assert len(seq) == len(server_seq)
+                assert all(seq[i] == server_seq[i] for i in range(len(seq)))
+            else:
+                expectA = 22 if source in UNREACHABLE_STATES else 0
+
+                get_server_shortest_path(source, target, expectA=expectA,
+                                         expectB=22)
+
+
+def test_migration_stop_copy_to_pre_copy_blocked():
+    global ctx, sock
+
+    feature = vfio_user_device_feature(
+        argsz=len(vfio_user_device_feature()) +
+            len(vfio_user_device_feature_mig_state()),
+        flags=VFIO_DEVICE_FEATURE_SET | VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE
+    )
+    payload = vfio_user_device_feature_mig_state(
+        device_state=VFIO_USER_DEVICE_STATE_STOP_COPY
+    )
+    msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload))
+
+    payload = vfio_user_device_feature_mig_state(
+        device_state=VFIO_USER_DEVICE_STATE_PRE_COPY
+    )
+    msg(ctx, sock, VFIO_USER_DEVICE_FEATURE, bytes(feature) + bytes(payload),
+        expect=22)
+
+
+def test_migration_cleanup():
+    disconnect_client(ctx, sock)
+    vfu_destroy_ctx(ctx)
+
+# ex: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab:

test/unit-tests.c

@@ -537,17 +537,6 @@ test_setup_migration_callbacks(void **state)
     assert_int_equal(p->v->migration->state, VFIO_USER_DEVICE_STATE_RUNNING);
 }
 
-static void
-test_setup_migration_callbacks_resuming(void **state)
-{
-    struct test_setup_migr_reg_dat *p = *state;
-    int r = vfu_setup_device_migration_callbacks(p->v,
-        LIBVFIO_USER_MIG_FLAG_START_RESUMING, &p->c);
-    assert_int_equal(0, r);
-    assert_non_null(p->v->migration);
-    assert_int_equal(p->v->migration->state, VFIO_USER_DEVICE_STATE_RESUMING);
-}
-
 static void
 test_handle_device_state(void **state)
 {
@@ -631,7 +620,7 @@ test_handle_mig_data_read_too_long(void **state) {
 
     migr->state = VFIO_USER_DEVICE_STATE_PRE_COPY;
     r = handle_mig_data_read(p->v, m);
-    assert_int_equal(-EINVAL, r);
+    assert_int_equal(-1, r);
 }
 
 static void
@@ -654,11 +643,11 @@ test_handle_mig_data_read_invalid_state(void **state) {
 
     migr->state = VFIO_USER_DEVICE_STATE_RUNNING;
     r = handle_mig_data_read(p->v, m);
-    assert_int_equal(-EINVAL, r);
+    assert_int_equal(-1, r);
 
     migr->state = VFIO_USER_DEVICE_STATE_STOP;
     r = handle_mig_data_read(p->v, m);
-    assert_int_equal(-EINVAL, r);
+    assert_int_equal(-1, r);
 }
 
 static void
@@ -719,7 +708,7 @@ test_handle_mig_data_write_invalid_state(void **state)
 
     migr->state = VFIO_USER_DEVICE_STATE_RUNNING;
     r = handle_mig_data_write(p->v, m);
-    assert_int_equal(-EINVAL, r);
+    assert_int_equal(-1, r);
 }
 
 static void
@@ -827,9 +816,6 @@ main(void)
         cmocka_unit_test_setup_teardown(test_setup_migration_callbacks,
             setup_test_setup_migration,
             teardown_test_setup_migration),
-        cmocka_unit_test_setup_teardown(test_setup_migration_callbacks_resuming,
-            setup_test_setup_migration,
-            teardown_test_setup_migration),
         cmocka_unit_test_setup_teardown(test_handle_device_state,
             setup_test_setup_migration,
             teardown_test_setup_migration),