Add integration tests for the stm32h743xx HIC's UDB flavour

UDB has a number of custom features that need to be tested before any UDB FW or HW release.

## How to run the tests?
All test are stadalone with no additional hardware setup required. Run these quick tests before committing changes.

Setup using: `python3 -m pip install -r "{path_to_test_folder}/requirements.txt"`.
Then run: `python udb_test_main.py --test_bin_path {path-to-binary-running-on-device-that-you-want-test} --dummy_bin_path {path-to-binary-that-has-a-different-version-from-the-test-image} --serial_port_path {path-to-serial-port}`
Example with full paths: `python source/hic_hal/stm32/stm32h743ii/extended_features/test/udb_test_main.py --test_bin_path ~/Downloads/test_0.12_local_stm32h743ii_udb_if_crc.bin --dummy_bin_path ~/Downloads/old_0.11_udb_stm32h743ii_if_crc.bin --serial_port_path /dev/serial/by-id/usb-Arm_DAPLink_CMSIS-DAP_00000081004400413330511331373438a5a5a5a597969940-if02`
You can additionally add -d for detailed logs and --run-all to run tests that require special attention.

## How to add new tests?
The test suite uses the standard python unittest library with the same standards. Add your tests to the test/udb_integration_test/tests folder. Each test needs to extend the TestCase class or a subclass of it. You can use any of the udb devices that are implemented to make it easier to talk to UDB through DAP, serial or the file system.

Before submitting your changes run mypy on the tests for type checking. Some libaries might have errors, but the test suite should have none.

Co-authored-by: Yangte Chen <yangtechen@google.com>
Co-authored-by: Eric Lee <eleenest@google.com>

test/udb_integration_test/README

@@ -0,0 +1,12 @@
+## How to run the tests?
+All test are stadalone with no additional hardware setup required. Run these quick tests before committing changes.
+
+Setup using: `python3 -m pip install -r "{path_to_test_folder}/requirements.txt"`.
+Then run: `python udb_test_main.py --test_bin_path {path-to-binary-running-on-device-that-you-want-test} --dummy_bin_path {path-to-binary-that-has-a-different-version-from-the-test-image} --serial_port_path {path-to-serial-port}`
+Example with full paths: `python source/hic_hal/stm32/stm32h743ii/extended_features/test/udb_test_main.py --test_bin_path ~/Downloads/test_0.12_local_stm32h743ii_udb_if_crc.bin --dummy_bin_path ~/Downloads/old_0.11_udb_stm32h743ii_if_crc.bin --serial_port_path /dev/serial/by-id/usb-Arm_DAPLink*-if04`
+You can additionally add -d for detailed logs and --run-all to run tests that require special attention.
+
+## How to add new tests?
+The test suite uses the standard python unittest library with the same standards. Add your tests to the test/udb_integration_test/tests folder. Each test needs to extend the TestCase class or a subclass of it. You can use any of the udb devices that are implemented to make it easier to talk to UDB through DAP, serial or the file system.
+
+Before submitting your changes run mypy on the tests for type checking. Some libaries might have errors, but the test suite should have none.

test/udb_integration_test/requirements.txt

@@ -0,0 +1,2 @@
+pyOCD==0.34.*
+pexpect

test/udb_integration_test/tests/test_dap_commands.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+from typing import ClassVar, Generator
+from udb_dap_device import UDBDapTestDevice
+from udb_test_helper import ContextTest
+from pyocd.probe.pydapaccess.cmsis_dap_core import Capabilities
+import logging
+
+logger = logging.getLogger("test.udb_integration_test")
+
+class DAPCommandTest(ContextTest):
+    udb: UDBDapTestDevice
+
+    def context(self) -> Generator:
+        with UDBDapTestDevice() as self.udb:
+            yield
+
+    def test_dap_cmd_info_for_swd_capability(self) -> None:
+        # Verify the DAP info command returned the expected data when the pyocd device was
+        # initialized
+        self.assertTrue((self.udb.get_device_dap_capabilities() & Capabilities.SWD) != 0,
+                        "No SWD capability returned in DAP info")
+
+    def test_dap_vendor_command_version(self) -> None:
+        # Verify device responds to vendor commands
+        self.assertTrue((self.udb.get_udb_interface_version()[1:5] == "udb_",
+                        "Wrong version returned"))

test/udb_integration_test/tests/test_shell_commands.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+from udb_test_helper import UDBTestResources, ContextTest
+from unittest import TestCase, skipUnless
+from udb_serial_device import UDBSerialTestDevice, OopsError
+from typing import Generator
+import time
+import re
+
+class ShellCommandTest(ContextTest):
+    udb_serial: UDBSerialTestDevice
+
+    def context(self) -> Generator:
+        with UDBSerialTestDevice() as self.udb_serial:
+            yield
+
+    def test_help(self) -> None:
+        output = self.udb_serial.command("help")
+        self.assertRegex(output, "show available commands")
+
+    def test_gpio(self) -> None:
+        output = self.udb_serial.command("gpio read E 9")
+        self.assertRegex(output, "GPIO port E pin 9 is 0")
+        output = self.udb_serial.command("gpio pp_set E 9")
+        self.assertRegex(output, "GPIO port E pin 9 is 1")
+        output = self.udb_serial.command("gpio read E 9")
+        self.assertRegex(output, "GPIO port E pin 9 is 1")
+        output = self.udb_serial.command("gpio pp_clear E 9")
+        self.assertRegex(output, "GPIO port E pin 9 is 0")
+        output = self.udb_serial.command("gpio read E 9")
+        self.assertRegex(output, "GPIO port E pin 9 is 0")
+
+    def test_pwm(self) -> None:
+        self.udb_serial.command("pwm start 50 60")
+        self.udb_serial.command("pwm stop")
+
+    def test_version(self) -> None:
+        output = self.udb_serial.command("version")
+        self.assertRegex(output, "Interface ver: udb_(.*)_hw:([0-9]*)\r\nBootloader " \
+                                 "ver: (.*)\r\n\r\nDONE 0\r")
+
+    def test_adapter_type(self) -> None:
+        self.udb_serial.command("adapter_type")
+
+    def test_i2c_probe(self) -> None:
+        output = self.udb_serial.command("i2c probe 2")
+        self.assertRegex(output, "probing...\r\n0x17\r\n0x50\r\n0x51\r\n0x52\r\n" \
+                                            "0x53\r\n0x54\r\n0x55\r\n0x56\r\n0x57")
+
+    def test_measure_power(self) -> None:
+        output = self.udb_serial.command("measure_power")
+        result = re.search("Target: Mainboard USB\r\n\tvoltage: ([0-9]*) mV\r\n\tcurrent: ([0-9]*) uA\r\n", output)
+        if result != None:
+            # needs this assert otherwise typing complains cause result is of type
+            # Optional[Match]
+            assert result is not None
+            self.assertLess(int(result.group(1)), 5150, "Voltage is unexpectedly large")
+            self.assertGreater(int(result.group(1)), 4850, "Voltage is unexpectedly small")
+            self.assertLess(int(result.group(2)), 180000, "Current is unexpectedly large")
+            self.assertGreater(int(result.group(2)), 113000, "Current is unexpectedly small")
+        else:
+            self.assertTrue(False, "Can't find expected output")
+
+    def test_uptime(self) -> None:
+        test_seconds = 10
+
+        output = self.udb_serial.command("uptime")
+        result = re.search("([0-9]*) mins ([0-9]*) secs\r\n", output)
+        prev_secs = int(result.group(1)) * 60 + int(result.group(2))
+
+        time.sleep(test_seconds)
+
+        output = self.udb_serial.command("uptime")
+        result = re.search("([0-9]*) mins ([0-9]*) secs\r\n", output)
+        secs = int(result.group(1)) * 60 + int(result.group(2))
+
+        # secs may wrap around in seconds
+        self.assertAlmostEqual((prev_secs + test_seconds) % 3600, secs, msg="uptime is not accurate", delta=1)
+
+    def test_ext_relay(self) -> None:
+        self.udb_serial.command("ext_relay on")
+        output = self.udb_serial.command("ext_relay status")
+        self.assertRegex(output, "external relay is on")
+        self.udb_serial.command("ext_relay off")
+        output = self.udb_serial.command("ext_relay status")
+        self.assertRegex(output, "external relay is off")
+
+    def test_swd_dut(self) -> None:
+        self.udb_serial.command("swd_dut 0")
+        output = self.udb_serial.command("swd_dut")
+        self.assertRegex(output, "DUT 0")
+        self.udb_serial.command("swd_dut 1")
+        output = self.udb_serial.command("swd_dut")
+        self.assertRegex(output, "DUT 1")
+
+    def test_btn(self) -> None:
+        for btn in ['RST0_L', 'BOOT0_L', 'BTN0_L', 'RST1', 'BOOT1', 'BTN1']:
+            self.udb_serial.command("btn {btn_name} press".format(btn_name=btn))
+            self.udb_serial.command("btn {btn_name} release".format(btn_name=btn))
+            self.udb_serial.command("btn {btn_name} tap".format(btn_name=btn))
+
+class ShellCommandWithResetTest(TestCase):
+    def test_reset(self) -> None:
+        with UDBSerialTestDevice() as udb_serial:
+            try:
+                output = udb_serial.command("reset")
+                self.assertTrue(False, f"Expected UDB to reset, but it didn't. Serial " \
+                                       f"output: {output}")
+            except OSError:
+                pass
+        with UDBSerialTestDevice() as udb_serial:
+            self.assertLess(udb_serial.get_time_to_open(),
+                            UDBTestResources.get_expected_boot_timedelta(),
+                            msg="Regression in boot time")
+
+    @skipUnless(UDBTestResources.should_run_all_tests(),
+                "this test runs only with the --run-all flag and you have to diconnect your " \
+                "debugger from UDB otherwise the assert will halt UDB")
+    def test_watchdog(self) -> None:
+        with UDBSerialTestDevice() as udb_serial:
+            try:
+                output = udb_serial.command("fault test_watchdog")
+                time.sleep(10)
+                self.assertTrue(False, f"Expected UDB to reset by watchdog, but it didn't. Serial " \
+                                       f"output: {output}")
+            except OSError:
+                pass
+        with UDBSerialTestDevice() as udb_serial:
+            self.assertLess(udb_serial.get_time_to_open(),
+                            UDBTestResources.get_expected_boot_timedelta(),
+                            msg="Regression in boot time")
+
+    @skipUnless(UDBTestResources.should_run_all_tests(),
+                "this test runs only with the --run-all flag and you have to diconnect your " \
+                "debugger from UDB otherwise the assert will halt UDB")
+    def test_assert(self) -> None:
+        with UDBSerialTestDevice() as udb_serial:
+            try:
+                output = udb_serial.command("fault test_assert")
+                self.assertTrue(False, "Expected UDB to reset, but it didn't. Please make sure " \
+                                       "there is no debugger connected to UDB, because then the " \
+                                       "assert will cause UDB to halt! Serial output: " \
+                                       f"{output}")
+            except (OopsError, OSError):
+                pass
+        with UDBSerialTestDevice() as udb_serial:
+            self.assertLess(udb_serial.get_time_to_open(),
+                            UDBTestResources.get_expected_boot_timedelta(),
+                            msg="Regression in boot time")

test/udb_integration_test/tests/test_software_update.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+from udb_serial_device import UDBSerialTestDevice
+from udb_dap_device import UDBDapTestDevice
+from datetime import datetime
+from udb_mass_storage_device import UDBMassStorageDevice
+from unittest import TestCase
+from udb_test_helper import UDBTestResources, indent_string
+import logging
+
+logger = logging.getLogger("test.udb_integration_test")
+
+class SoftwareUpdateTest(TestCase):
+    def test_swu(self) -> None:
+        with UDBDapTestDevice() as udb_dap:
+            version_1 = udb_dap.get_udb_interface_version()
+            logger.info("\n" + indent_string(f"Version before test: {version_1}"))
+        with UDBSerialTestDevice() as udb:
+            logger.info(indent_string("Resetting into SWU mode..."))
+            udb.command_no_wait("\nreset_into_swu_mode")
+        with UDBMassStorageDevice() as udb:
+            start = datetime.now()
+            logger.info(indent_string("Copying dummy binary with different version..."))
+            udb.copy_firmware(UDBTestResources.get_path_to_binary_with_diff_version())
+        with UDBSerialTestDevice() as udb:
+            self.assertLess(udb.get_time_to_open(),
+                            UDBTestResources.get_expected_boot_timedelta(),
+                            msg="Regression in boot time")
+        with UDBDapTestDevice() as udb_dap:
+            version_2 = udb_dap.get_udb_interface_version()
+            swu_time_taken = datetime.now() - start
+            logger.info(indent_string(f"Version after update:{version_2}"))
+        with UDBSerialTestDevice() as udb:
+            logger.info(indent_string("Resetting into SWU mode..."))
+            udb.command_no_wait("\nreset_into_swu_mode")
+        with UDBMassStorageDevice() as udb:
+            logger.info(indent_string("Copying back the original test binary..."))
+            udb.copy_firmware(UDBTestResources.get_path_to_current_binary())
+        with UDBSerialTestDevice() as udb:
+            self.assertLess(udb.get_time_to_open(),
+                            UDBTestResources.get_expected_boot_timedelta(),
+                            msg="Regression in boot time")
+        with UDBDapTestDevice() as udb_dap:
+            version_3 = udb_dap.get_udb_interface_version()
+            logger.info(indent_string(f"Version after the test:{version_3}"))
+
+        logger.info(indent_string(f"The software update test took {swu_time_taken.seconds}s"))
+
+        expected_swu_time_sec = 30
+        self.assertLess(swu_time_taken.seconds, expected_swu_time_sec, f"SWU took too long")
+        self.assertEqual(version_1, version_3, "The firmware version is not the same after the " \
+                         "test as before the test. You probably provide the wrong binary.")
+        self.assertNotEqual(version_1, version_2, "The version after the test software update " \
+                            "is the same as before, SWU probably failed or the wrong dummy " \
+                            "binary was provided.")

test/udb_integration_test/tests/test_usb_stress.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+from typing import Generator
+from udb_serial_device import UDBSerialTestDevice
+from udb_dap_device import UDBDapTestDevice
+from datetime import datetime
+from unittest import skipUnless
+import logging
+from udb_test_helper import UDBTestResources, ContextTest, indent_string
+
+logger = logging.getLogger("test.udb_integration_test")
+
+class USBStressTest(ContextTest):
+    udb_dap: UDBDapTestDevice
+    udb_serial: UDBSerialTestDevice
+
+    def context(self) -> Generator:
+        with UDBSerialTestDevice(baudrate=3000000) as self.udb_serial:
+            with UDBDapTestDevice() as self.udb_dap:
+                yield
+
+    def test_usb(self) -> None:
+        start = datetime.now()
+        count = 0
+        while True:
+            self.udb_serial.command_no_wait("TEST1TEST2TEST3TEST4TEST5TEST6TEST7TEST8TEST9TEST" \
+                                            "TEST1TEST2TEST3TEST4TEST5TEST6TEST7TEST8TEST9TEST")
+            self.assertEqual(self.udb_dap.get_udb_interface_version()[1:5],
+                             "udb_", "DAP commandreplies are bad")
+            count += 1
+            if (datetime.now() - start).seconds > 15:
+                break
+        self.udb_serial.flush()
+        output = self.udb_serial.command("help")
+        self.assertRegex(output, "show available commands")
+        logger.info("\n" + indent_string(f"Read and wrote {count} times..."))

test/udb_integration_test/udb_dap_device.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+from typing import ClassVar
+from pyocd.probe import pydapaccess
+from datetime import datetime, timedelta
+import logging
+from udb_test_helper import UDBTestResources
+
+logger = logging.getLogger("test.udb_integration_test")
+
+class DapDeviceError(Exception):
+    pass
+
+class UDBDapTestDevice:
+    timeout: ClassVar[timedelta] = timedelta(seconds=0.5)
+
+    def __enter__(self):
+        self.device = None
+        unique_id = None
+        start = datetime.now()
+        while unique_id == None:
+            for dev in pydapaccess.DAPAccess.get_connected_devices():
+                if dev.product_name[0:7] == "DAPLink":
+                    if unique_id is None:
+                        unique_id = dev.get_unique_id()
+                    else:
+                        logger.warning("WARNING: multiple DAPLinks are connected! Picking the" \
+                                       " first one...")
+            if datetime.now() - start > UDBTestResources.get_expected_boot_timedelta():
+                raise DapDeviceError("Timeout: Can't find any DAPLink device connected")
+
+        self.device = pydapaccess.DAPAccess.get_device(unique_id)
+        if self.device is None:
+            raise DapDeviceError("Can't get DAPLink device object")
+
+        self.device.open()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.device.close()
+
+    def run_dap_test(self,
+                     command_number: int,
+                     send_data: list[int],
+                     expected_data: list[int]=None,
+                     expected_data_bit_mask: list[int]=None) -> list:
+        result = self.device.vendor(command_number, send_data)
+        logger.debug("Result: ", result)
+        if expected_data and expected_data_bit_mask:
+            # Not yet implemented
+            assert False
+        return result
+
+    def get_device_dap_capabilities(self) -> int:
+        return self.device._capabilities
+
+    def get_udb_interface_version(self) -> str:
+        vendor_cmd_id_get_version = 36
+        result = self.device.vendor(vendor_cmd_id_get_version, [0])
+        version = "".join(map(chr, result))
+        logger.debug("".join(map(chr, result)))
+        return version