Merge pull request #38 from LedgerHQ/xch/ragger-conftest

tests: Update conftests with latests improvments

.github/workflows/ci-workflow.yml

@@ -113,6 +113,12 @@ jobs:
  name: Functional tests with Ragger (compatible with Speculos & physical devices)
  needs:
  - nano_build
+ strategy:
+ matrix:
+ include:
+ - model: nanos
+ - model: nanox
+ - model: nanosp
  runs-on: ubuntu-latest
  steps:
  - name: Clone
@@ -129,4 +135,4 @@ jobs:
  - name: Run test
  env:
  CTEST_OUTPUT_ON_FAILURE: 1
- run: pytest --tb=short -v tests/
+ run: pytest --tb=short -v tests/ --device ${{ matrix.model }}

tests/conftest.py

@@ -1,4 +1,5 @@
 import pytest
+from typing import Optional
 from pathlib import Path
 from ragger.firmware import Firmware
 from ragger.backend import SpeculosBackend, LedgerCommBackend, LedgerWalletBackend
@@ -10,24 +11,24 @@
 # Adapt this path to your 'tests/elfs' directory
 APPS_DIRECTORY = (Path(__file__).parent / "elfs").resolve()
 
-# Adapt this path to the APPNAME in your Makefile
+# Adapt this name part of the compiled app <name>_<device>.elf in the APPS_DIRECTORY
 APP_NAME = "boilerplate"
 
 BACKENDS = ["speculos", "ledgercomm", "ledgerwallet"]
 
+DEVICES = ["nanos", "nanox", "nanosp", "all"]
+
 FIRMWARES = [Firmware('nanos', '2.1'),
  Firmware('nanox', '2.0.2'),
  Firmware('nanosp', '1.0.3')]
 
 
 def pytest_addoption(parser):
- # the default backend is Speculos, this option allows to select another
- parser.addoption("--backend", action="store", default="speculos")
+ parser.addoption("--device", choices=DEVICES, required=True)
+ parser.addoption("--backend", choices=BACKENDS, default="speculos")
  parser.addoption("--display", action="store_true", default=False)
  parser.addoption("--golden_run", action="store_true", default=False)
- # Enable using --'device' in the pytest command line to restrict testing to specific devices
- for fw in FIRMWARES:
- parser.addoption("--"+fw.device, action="store_true", help="run on nanos only")
+ parser.addoption("--log_apdu_file", action="store", default=None)
 
 
 @pytest.fixture(scope="session")
@@ -45,6 +46,12 @@ def golden_run(pytestconfig):
  return pytestconfig.getoption("golden_run")
 
 
+@pytest.fixture(scope="session")
+def log_apdu_file(pytestconfig):
+ filename = pytestconfig.getoption("log_apdu_file")
+ return Path(filename).resolve() if filename is not None else None
+
+
 @pytest.fixture
 def test_name(request):
  # Get the name of current pytest test
@@ -61,17 +68,27 @@ def pytest_generate_tests(metafunc):
  if "firmware" in metafunc.fixturenames:
  fw_list = []
  ids = []
- # First pass: enable only demanded firmwares
- for fw in FIRMWARES:
- if metafunc.config.getoption(fw.device):
- fw_list.append(fw)
- ids.append(fw.device + " " + fw.version)
- # Second pass if no specific firmware demanded: add them all
- if not fw_list:
+
+ device = metafunc.config.getoption("device")
+ backend_name = metafunc.config.getoption("backend")
+
+ if device == "all":
+ if backend_name != "speculos":
+ raise ValueError("Invalid device parameter on this backend")
+
+ # Add all supported firmwares
  for fw in FIRMWARES:
  fw_list.append(fw)
  ids.append(fw.device + " " + fw.version)
- metafunc.parametrize("firmware", fw_list, ids=ids)
+
+ else:
+ # Enable firmware for demanded device
+ for fw in FIRMWARES:
+ if device == fw.device:
+ fw_list.append(fw)
+ ids.append(fw.device + " " + fw.version)
+
+ metafunc.parametrize("firmware", fw_list, ids=ids, scope="session")
 
 
 def prepare_speculos_args(firmware: Firmware, display: bool):
@@ -88,22 +105,26 @@ def prepare_speculos_args(firmware: Firmware, display: bool):
 # Depending on the "--backend" option value, a different backend is
 # instantiated, and the tests will either run on Speculos or on a physical
 # device depending on the backend
-def create_backend(backend_name: str, firmware: Firmware, display: bool):
+def create_backend(backend_name: str, firmware: Firmware, display: bool, log_apdu_file: Optional[Path]):
  if backend_name.lower() == "ledgercomm":
- return LedgerCommBackend(firmware, interface="hid")
+ return LedgerCommBackend(firmware=firmware, interface="hid", log_apdu_file=log_apdu_file)
  elif backend_name.lower() == "ledgerwallet":
- return LedgerWalletBackend(firmware)
+ return LedgerWalletBackend(firmware=firmware, log_apdu_file=log_apdu_file)
  elif backend_name.lower() == "speculos":
  args, kwargs = prepare_speculos_args(firmware, display)
- return SpeculosBackend(*args, firmware, **kwargs)
+ return SpeculosBackend(*args, firmware=firmware, log_apdu_file=log_apdu_file, **kwargs)
  else:
  raise ValueError(f"Backend '{backend_name}' is unknown. Valid backends are: {BACKENDS}")
 
 
-# This final fixture will return the properly configured backend, to be used in tests
-@pytest.fixture
-def backend(backend_name, firmware, display):
- with create_backend(backend_name, firmware, display) as b:
+# This fixture will return the properly configured backend, to be used in tests.
+# As Speculos instantiation takes some time, this fixture scope is by default "session".
+# If your tests needs to be run on independent Speculos instances (in case they affect
+# settings for example), then you should change this fixture scope and choose between
+# function, class, module or session.
+@pytest.fixture(scope="session")
+def backend(backend_name, firmware, display, log_apdu_file):
+ with create_backend(backend_name, firmware, display, log_apdu_file) as b:
  yield b
 
 
@@ -120,7 +141,7 @@ def use_only_on_backend(request, backend):
  if request.node.get_closest_marker('use_on_backend'):
  current_backend = request.node.get_closest_marker('use_on_backend').args[0]
  if current_backend != backend:
- pytest.skip('skipped on this backend: {}'.format(current_backend))
+ pytest.skip(f'skipped on this backend: "{current_backend}"')
 
 
 def pytest_configure(config):

tests/requirements.txt

@@ -1,6 +1,6 @@
 pytest
-ragger[speculos]>=1.1.0
-ragger[ledgerwallet]>=1.1.0
+ragger[speculos]>=1.2.0
+ragger[ledgerwallet]>=1.2.0
 Jinja2==3.1.2
 Flask==2.1.2
 ecdsa>=0.16.1,<0.17.0

tests/usage.md

@@ -36,7 +36,7 @@ cp bin/app.elf tests/elfs/<appname>_<device>.elf # replace <device> with one
 
 You can use the following command to get your first experience with Ragger and Speculos
 ```
-pytest -v --tb=short --nanox --display
+pytest -v --tb=short --device nanox --display
 ```
 Or you can refer to the section `Available pytest options` to configure the options you want to use
 
@@ -56,7 +56,7 @@ exit
 You can use the following command to get your first experience with Ragger and Ledgerwallet on a NANOX.
 Make sure that the device is plugged, unlocked, and that the tested application is open.
 ```
-pytest -v --tb=short --nanox --backend ledgerwallet
+pytest -v --tb=short --device nanox --backend ledgerwallet
 ```
 Or you can refer to the section `Available pytest options` to configure the options you want to use
 
@@ -73,11 +73,9 @@ Standard useful pytest options
 
 Custom pytest options
 ```
- --backend <backend> run the tests against the backend [speculos, ledgercomm, ledgerwallet]. Speculos is the default
- --display on Speculos, enables the display of the app screen using QT
- --golden_run on Speculos, screen comparison functions will save the current screen instead of comparing
- --nanos run only the test for the nanos device
- --nanox run only the test for the nanox device
- --nanosp run only the test for the nanosp device
+ --device <device> run the test on the specified device [nanos,nanox,nanosp,all]. This parameter is mandatory
+ --backend <backend> run the tests against the backend [speculos, ledgercomm, ledgerwallet]. Speculos is the default
+ --display on Speculos, enables the display of the app screen using QT
+ --golden_run on Speculos, screen comparison functions will save the current screen instead of comparing
+ --log_apdu_file <filepath> log all apdu exchanges to the file in parameter. The previous file content is erased
 ``` 
-