diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 4ac2b5f35..eb8d0631d 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -113,6 +113,12 @@ jobs: name: Functional tests with Ragger (compatible with Speculos & physical devices) needs: - nano_build + strategy: + matrix: + include: + - model: nanos + - model: nanox + - model: nanosp runs-on: ubuntu-latest steps: - name: Clone @@ -129,4 +135,4 @@ jobs: - name: Run test env: CTEST_OUTPUT_ON_FAILURE: 1 - run: pytest --tb=short -v tests/ + run: pytest --tb=short -v tests/ --device ${{ matrix.model }} diff --git a/tests/conftest.py b/tests/conftest.py index b0d1e6fd0..c0e3db35d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import pytest +from typing import Optional from pathlib import Path from ragger.firmware import Firmware from ragger.backend import SpeculosBackend, LedgerCommBackend, LedgerWalletBackend @@ -10,24 +11,24 @@ # Adapt this path to your 'tests/elfs' directory APPS_DIRECTORY = (Path(__file__).parent / "elfs").resolve() -# Adapt this path to the APPNAME in your Makefile +# Adapt this name part of the compiled app _.elf in the APPS_DIRECTORY APP_NAME = "boilerplate" BACKENDS = ["speculos", "ledgercomm", "ledgerwallet"] +DEVICES = ["nanos", "nanox", "nanosp", "all"] + FIRMWARES = [Firmware('nanos', '2.1'), Firmware('nanox', '2.0.2'), Firmware('nanosp', '1.0.3')] def pytest_addoption(parser): - # the default backend is Speculos, this option allows to select another - parser.addoption("--backend", action="store", default="speculos") + parser.addoption("--device", choices=DEVICES, required=True) + parser.addoption("--backend", choices=BACKENDS, default="speculos") parser.addoption("--display", action="store_true", default=False) parser.addoption("--golden_run", action="store_true", default=False) - # Enable using --'device' in the pytest command line to restrict testing to specific devices - for fw in FIRMWARES: - parser.addoption("--"+fw.device, action="store_true", help="run on nanos only") + parser.addoption("--log_apdu_file", action="store", default=None) @pytest.fixture(scope="session") @@ -45,6 +46,12 @@ def golden_run(pytestconfig): return pytestconfig.getoption("golden_run") +@pytest.fixture(scope="session") +def log_apdu_file(pytestconfig): + filename = pytestconfig.getoption("log_apdu_file") + return Path(filename).resolve() if filename is not None else None + + @pytest.fixture def test_name(request): # Get the name of current pytest test @@ -61,17 +68,27 @@ def pytest_generate_tests(metafunc): if "firmware" in metafunc.fixturenames: fw_list = [] ids = [] - # First pass: enable only demanded firmwares - for fw in FIRMWARES: - if metafunc.config.getoption(fw.device): - fw_list.append(fw) - ids.append(fw.device + " " + fw.version) - # Second pass if no specific firmware demanded: add them all - if not fw_list: + + device = metafunc.config.getoption("device") + backend_name = metafunc.config.getoption("backend") + + if device == "all": + if backend_name != "speculos": + raise ValueError("Invalid device parameter on this backend") + + # Add all supported firmwares for fw in FIRMWARES: fw_list.append(fw) ids.append(fw.device + " " + fw.version) - metafunc.parametrize("firmware", fw_list, ids=ids) + + else: + # Enable firmware for demanded device + for fw in FIRMWARES: + if device == fw.device: + fw_list.append(fw) + ids.append(fw.device + " " + fw.version) + + metafunc.parametrize("firmware", fw_list, ids=ids, scope="session") def prepare_speculos_args(firmware: Firmware, display: bool): @@ -88,22 +105,26 @@ def prepare_speculos_args(firmware: Firmware, display: bool): # Depending on the "--backend" option value, a different backend is # instantiated, and the tests will either run on Speculos or on a physical # device depending on the backend -def create_backend(backend_name: str, firmware: Firmware, display: bool): +def create_backend(backend_name: str, firmware: Firmware, display: bool, log_apdu_file: Optional[Path]): if backend_name.lower() == "ledgercomm": - return LedgerCommBackend(firmware, interface="hid") + return LedgerCommBackend(firmware=firmware, interface="hid", log_apdu_file=log_apdu_file) elif backend_name.lower() == "ledgerwallet": - return LedgerWalletBackend(firmware) + return LedgerWalletBackend(firmware=firmware, log_apdu_file=log_apdu_file) elif backend_name.lower() == "speculos": args, kwargs = prepare_speculos_args(firmware, display) - return SpeculosBackend(*args, firmware, **kwargs) + return SpeculosBackend(*args, firmware=firmware, log_apdu_file=log_apdu_file, **kwargs) else: raise ValueError(f"Backend '{backend_name}' is unknown. Valid backends are: {BACKENDS}") -# This final fixture will return the properly configured backend, to be used in tests -@pytest.fixture -def backend(backend_name, firmware, display): - with create_backend(backend_name, firmware, display) as b: +# This fixture will return the properly configured backend, to be used in tests. +# As Speculos instantiation takes some time, this fixture scope is by default "session". +# If your tests needs to be run on independent Speculos instances (in case they affect +# settings for example), then you should change this fixture scope and choose between +# function, class, module or session. +@pytest.fixture(scope="session") +def backend(backend_name, firmware, display, log_apdu_file): + with create_backend(backend_name, firmware, display, log_apdu_file) as b: yield b @@ -120,7 +141,7 @@ def use_only_on_backend(request, backend): if request.node.get_closest_marker('use_on_backend'): current_backend = request.node.get_closest_marker('use_on_backend').args[0] if current_backend != backend: - pytest.skip('skipped on this backend: {}'.format(current_backend)) + pytest.skip(f'skipped on this backend: "{current_backend}"') def pytest_configure(config): diff --git a/tests/requirements.txt b/tests/requirements.txt index 6ed29c284..17e5b6f88 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,6 @@ pytest -ragger[speculos]>=1.1.0 -ragger[ledgerwallet]>=1.1.0 +ragger[speculos]>=1.2.0 +ragger[ledgerwallet]>=1.2.0 Jinja2==3.1.2 Flask==2.1.2 ecdsa>=0.16.1,<0.17.0 diff --git a/tests/usage.md b/tests/usage.md index 2f677ca77..ecaffd4b2 100644 --- a/tests/usage.md +++ b/tests/usage.md @@ -36,7 +36,7 @@ cp bin/app.elf tests/elfs/_.elf # replace with one You can use the following command to get your first experience with Ragger and Speculos ``` -pytest -v --tb=short --nanox --display +pytest -v --tb=short --device nanox --display ``` Or you can refer to the section `Available pytest options` to configure the options you want to use @@ -56,7 +56,7 @@ exit You can use the following command to get your first experience with Ragger and Ledgerwallet on a NANOX. Make sure that the device is plugged, unlocked, and that the tested application is open. ``` -pytest -v --tb=short --nanox --backend ledgerwallet +pytest -v --tb=short --device nanox --backend ledgerwallet ``` Or you can refer to the section `Available pytest options` to configure the options you want to use @@ -73,11 +73,9 @@ Standard useful pytest options Custom pytest options ``` - --backend run the tests against the backend [speculos, ledgercomm, ledgerwallet]. Speculos is the default - --display on Speculos, enables the display of the app screen using QT - --golden_run on Speculos, screen comparison functions will save the current screen instead of comparing - --nanos run only the test for the nanos device - --nanox run only the test for the nanox device - --nanosp run only the test for the nanosp device + --device run the test on the specified device [nanos,nanox,nanosp,all]. This parameter is mandatory + --backend run the tests against the backend [speculos, ledgercomm, ledgerwallet]. Speculos is the default + --display on Speculos, enables the display of the app screen using QT + --golden_run on Speculos, screen comparison functions will save the current screen instead of comparing + --log_apdu_file log all apdu exchanges to the file in parameter. The previous file content is erased ``` -