Performance test suite

.github/workflows/ci-workflow.yml

@@ -53,6 +53,28 @@ jobs:
  name: bitcoin-testnet-app-${{ matrix.model }}
  path: bitcoin-testnet-bin
 
+ job_build_app_perftest:
+ name: Compile the UX-less version of the app on Nano S+ for performance tests
+
+ runs-on: ubuntu-latest
+
+ container:
+ image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v2
+
+ - name: Build
+ run: |
+ make DEBUG=0 COIN=bitcoin_testnet BOLOS_SDK="$NANOSP_SDK" AUTOAPPROVE_FOR_PERF_TESTS=1
+
+ - name: Upload Bitcoin Testnet app binary
+ uses: actions/upload-artifact@v2
+ with:
+ name: bitcoin-testnet-perftest-app-nanosp
+ path: bin
+
  job_unit_test:
  name: Unit test
  needs: job_build
@@ -166,6 +188,43 @@ jobs:
  pip install -r requirements.txt
  PYTHONPATH=$PYTHONPATH:/speculos pytest --headless --timeout=300
 
+ job_perftests:
+ name: Performance report
+
+ needs: job_build_app_perftest
+ runs-on: ubuntu-latest
+
+ container:
+ image: ghcr.io/ledgerhq/app-bitcoin-new/speculos-bitcoin:latest
+ ports:
+ - 1234:1234
+ - 9999:9999
+ - 40000:40000
+ - 41000:41000
+ - 42000:42000
+ - 43000:43000
+ options: --entrypoint /bin/bash
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v2
+
+ - name: Download Bitcoin app binary for perftests
+ uses: actions/download-artifact@v2
+ with:
+ name: bitcoin-testnet-perftest-app-nanosp
+ path: bin
+
+ - name: Run tests
+ run: |
+ cd tests_perf
+ pip install -r requirements.txt
+ PYTHONPATH=$PYTHONPATH:/speculos pytest --headless --model=nanosp --benchmark-json=benchmarks.json
+ - name: Upload benchmarks summary
+ uses: actions/upload-artifact@v2
+ with:
+ name: benchmarks-log
+ path: tests_perf/benchmarks.json
 
  job_test_js_lib:
  name: Tests with the JS library

Makefile

@@ -56,6 +56,13 @@ else
 APPVERSION = "$(APPVERSION_M).$(APPVERSION_N).$(APPVERSION_P)-$(strip $(APPVERSION_SUFFIX))"
 endif
 
+# If set, the app will automatically approve all requests without user interaction. Useful for performance tests.
+# It is critical that no such app is ever deployed in production.
+AUTOAPPROVE_FOR_PERF_TESTS ?= 0
+ifneq ($(AUTOAPPROVE_FOR_PERF_TESTS),0)
+ DEFINES += HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+endif
+
 # Setting to allow building variant applications
 VARIANT_PARAM = COIN
 VARIANT_VALUES = bitcoin_testnet bitcoin
@@ -74,33 +81,36 @@ HAVE_APPLICATION_FLAG_BOLOS_SETTINGS = 1
 HAVE_APPLICATION_FLAG_LIBRARY = 1
 
 ifeq ($(COIN),bitcoin_testnet)
-
-# Bitcoin testnet, no legacy support
-DEFINES += BIP32_PUBKEY_VERSION=0x043587CF
-DEFINES += BIP44_COIN_TYPE=1
-DEFINES += COIN_P2PKH_VERSION=111
-DEFINES += COIN_P2SH_VERSION=196
-DEFINES += COIN_NATIVE_SEGWIT_PREFIX=\"tb\"
-DEFINES += COIN_COINID_SHORT=\"TEST\"
-
-APPNAME = "Bitcoin Test"
-
+ # Bitcoin testnet, no legacy support
+ DEFINES += BIP32_PUBKEY_VERSION=0x043587CF
+ DEFINES += BIP44_COIN_TYPE=1
+ DEFINES += COIN_P2PKH_VERSION=111
+ DEFINES += COIN_P2SH_VERSION=196
+ DEFINES += COIN_NATIVE_SEGWIT_PREFIX=\"tb\"
+ DEFINES += COIN_COINID_SHORT=\"TEST\"
+
+ APPNAME = "Bitcoin Test"
 else ifeq ($(COIN),bitcoin)
-
-# Bitcoin mainnet, no legacy support
-DEFINES += BIP32_PUBKEY_VERSION=0x0488B21E
-DEFINES += BIP44_COIN_TYPE=0
-DEFINES += COIN_P2PKH_VERSION=0
-DEFINES += COIN_P2SH_VERSION=5
-DEFINES += COIN_NATIVE_SEGWIT_PREFIX=\"bc\"
-DEFINES += COIN_COINID_SHORT=\"BTC\"
-
-APPNAME = "Bitcoin"
+ # the version for performance tests automatically approves all requests
+ # there is no reason to ever compile the mainnet app with this flag
+ ifneq ($(AUTOAPPROVE_FOR_PERF_TESTS),0)
+ $(error Use testnet app for performance tests)
+ endif
+
+ # Bitcoin mainnet, no legacy support
+ DEFINES += BIP32_PUBKEY_VERSION=0x0488B21E
+ DEFINES += BIP44_COIN_TYPE=0
+ DEFINES += COIN_P2PKH_VERSION=0
+ DEFINES += COIN_P2SH_VERSION=5
+ DEFINES += COIN_NATIVE_SEGWIT_PREFIX=\"bc\"
+ DEFINES += COIN_COINID_SHORT=\"BTC\"
+
+ APPNAME = "Bitcoin"
 
 else
-ifeq ($(filter clean,$(MAKECMDGOALS)),)
-$(error Unsupported COIN - use bitcoin_testnet, bitcoin)
-endif
+ ifeq ($(filter clean,$(MAKECMDGOALS)),)
+ $(error Unsupported COIN - use bitcoin_testnet, bitcoin)
+ endif
 endif
 
 # Application icons following guidelines:

src/handler/sign_message.c

@@ -174,6 +174,7 @@ void handler_sign_message(dispatcher_context_t* dc, uint8_t protocol_version) {
  snprintf(message_hash_str + 2 * i, 3, "%02X", message_hash[i]);
  }
 
+#ifndef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
  ui_pre_processing_message();
  if (printable) {
  if (!display_message_content_and_confirm(dc,
@@ -189,6 +190,7 @@ void handler_sign_message(dispatcher_context_t* dc, uint8_t protocol_version) {
  return;
  }
  }
+#endif
  uint8_t sig[MAX_DER_SIG_LEN];
 
  uint32_t info;

src/ui/display.c

@@ -93,6 +93,10 @@ bool ui_display_pubkey(dispatcher_context_t *context,
  const char *bip32_path_str,
  bool is_path_suspicious,
  const char *pubkey) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_path_and_pubkey_state_t *state = (ui_path_and_pubkey_state_t *) &g_ui_state;
 
  strncpy(state->bip32_path_str, bip32_path_str, sizeof(state->bip32_path_str));
@@ -110,6 +114,10 @@ bool ui_display_pubkey(dispatcher_context_t *context,
 bool ui_display_path_and_message_content(dispatcher_context_t *context,
  const char *path_str,
  const char *message_content) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_path_and_message_state_t *state = (ui_path_and_message_state_t *) &g_ui_state;
  strncpy(state->bip32_path_str, path_str, sizeof(state->bip32_path_str));
  strncpy(state->message, message_content, sizeof(state->message));
@@ -122,6 +130,10 @@ bool ui_display_path_and_message_content(dispatcher_context_t *context,
 bool ui_display_message_path_hash_and_confirm(dispatcher_context_t *context,
  const char *path_str,
  const char *message_hash) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_path_and_message_state_t *state = (ui_path_and_message_state_t *) &g_ui_state;
  strncpy(state->bip32_path_str, path_str, sizeof(state->bip32_path_str));
  strncpy(state->message, message_hash, sizeof(state->message));
@@ -132,6 +144,10 @@ bool ui_display_message_path_hash_and_confirm(dispatcher_context_t *context,
 }
 
 bool ui_display_message_confirm(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  (void) context;
  ui_sign_message_confirm_flow();
 
@@ -141,6 +157,10 @@ bool ui_display_message_confirm(dispatcher_context_t *context) {
 bool ui_display_register_wallet(dispatcher_context_t *context,
  const policy_map_wallet_header_t *wallet_header,
  const char *policy_descriptor) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_wallet_state_t *state = (ui_wallet_state_t *) &g_ui_state;
 
  strncpy(state->wallet_name, wallet_header->name, sizeof(state->wallet_name));
@@ -158,6 +178,10 @@ bool ui_display_policy_map_cosigner_pubkey(dispatcher_context_t *context,
  uint8_t cosigner_index,
  uint8_t n_keys,
  key_type_e key_type) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  (void) (n_keys);
 
  ui_cosigner_pubkey_and_index_state_t *state =
@@ -190,6 +214,10 @@ bool ui_display_wallet_address(dispatcher_context_t *context,
  const char *address) {
  ui_wallet_state_t *state = (ui_wallet_state_t *) &g_ui_state;
 
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  strncpy(state->address, address, sizeof(state->address));
 
  if (wallet_name == NULL) {
@@ -205,28 +233,48 @@ bool ui_display_wallet_address(dispatcher_context_t *context,
 bool ui_authorize_wallet_spend(dispatcher_context_t *context, const char *wallet_name) {
  ui_wallet_state_t *state = (ui_wallet_state_t *) &g_ui_state;
 
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  strncpy(state->wallet_name, wallet_name, sizeof(state->wallet_name));
  ui_display_spend_from_wallet_flow();
 
  return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_warn_external_inputs(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_display_warning_external_inputs_flow();
  return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_warn_unverified_segwit_inputs(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_display_unverified_segwit_inputs_flows();
  return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_warn_nondefault_sighash(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_display_nondefault_sighash_flow();
  return io_ui_process(context, SET_UX_DIRTY);
 }
 
 bool ui_transaction_prompt(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_display_transaction_prompt();
  return io_ui_process(context, SET_UX_DIRTY);
 }
@@ -237,6 +285,10 @@ bool ui_validate_output(dispatcher_context_t *context,
  const char *address_or_description,
  const char *coin_name,
  uint64_t amount) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_validate_output_state_t *state = (ui_validate_output_state_t *) &g_ui_state;
 
  strncpy(state->address_or_description,
@@ -254,6 +306,10 @@ bool ui_validate_output(dispatcher_context_t *context,
 }
 
 bool ui_warn_high_fee(dispatcher_context_t *context) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_warn_high_fee_flow();
 
  return io_ui_process(context, SET_UX_DIRTY);
@@ -263,6 +319,10 @@ bool ui_validate_transaction(dispatcher_context_t *context,
  const char *coin_name,
  uint64_t fee,
  bool is_self_transfer) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  ui_validate_transaction_state_t *state = (ui_validate_transaction_state_t *) &g_ui_state;
 
  format_sats_amount(coin_name, fee, state->fee);
@@ -298,20 +358,32 @@ void ui_pre_processing_message(void) {
 
 #ifdef HAVE_NBGL
 bool ui_post_processing_confirm_wallet_registration(dispatcher_context_t *context, bool success) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  (void) context;
  ui_display_post_processing_confirm_wallet_registation(success);
 
  return true;
 }
 
 bool ui_post_processing_confirm_transaction(dispatcher_context_t *context, bool success) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  (void) context;
  ui_display_post_processing_confirm_transaction(success);
 
  return true;
 }
 
 bool ui_post_processing_confirm_message(dispatcher_context_t *context, bool success) {
+#ifdef HAVE_AUTOAPPROVE_FOR_PERF_TESTS
+ return true;
+#endif
+
  (void) context;
  ui_display_post_processing_confirm_message(success);